From 61fbef42a6eeae7553f148f1759c5a770a2f65aa Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 18 Oct 2025 18:47:27 +0300 Subject: [PATCH 001/155] libstore: Simplify check for S3-specific URI query parameters Instead of hardcoding strings we should instead use the setting objects to determine the query names that should be preserved. --- .../include/nix/store/s3-binary-cache-store.hh | 8 ++++++-- src/libstore/s3-binary-cache-store.cc | 15 +++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index c8cb967c1..e5fcbeda3 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -21,8 +21,6 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig Nix uses the `default` profile. )"}; -public: - const Setting region{ this, "us-east-1", @@ -63,6 +61,12 @@ public: > addressing instead of virtual host based addressing. )"}; + /** + * Set of settings that are part of the S3 URI itself. + * These are needed for region specification and other S3-specific settings. + */ + const std::set s3UriSettings = {&profile, ®ion, &scheme, &endpoint}; + static const std::string name() { return "S3 Binary Cache Store"; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index a84ea5fcb..ac08a4982 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,10 +1,10 @@ #include "nix/store/s3-binary-cache-store.hh" - -#include - #include "nix/store/http-binary-cache-store.hh" #include "nix/store/store-registration.hh" +#include +#include + namespace nix { StringSet S3BinaryCacheStoreConfig::uriSchemes() @@ -17,14 +17,13 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( : StoreConfig(params) , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params) { - // For S3 stores, preserve S3-specific query parameters as part of the URL - // These are needed for region specification and other S3-specific settings assert(cacheUri.query.empty()); + assert(cacheUri.scheme == "s3"); - // Only copy S3-specific parameters to the URL query - static const std::set s3Params = {"region", "endpoint", "profile", "scheme"}; for (const auto & [key, value] : params) { - if (s3Params.contains(key)) { + auto s3Params = + std::views::transform(s3UriSettings, [](const AbstractSetting * setting) { return setting->name; }); + if (std::ranges::contains(s3Params, key)) { cacheUri.query[key] = value; } } From 3d147c04a5f9d03e1696fb25b495a077885d2cf7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 18 Oct 2025 19:11:39 +0300 Subject: [PATCH 002/155] libstore: Implement getHumanReadableURI for S3BinaryCacheStoreConfig This slightly improves the logs situation by including the region/profile/endpoint in the logs when S3 store references get printed. Instead of: copying path '/nix/store/lxnp9cs4cfh2g9r2bs4z7gwwz9kdj2r9-test-package-c' to 's3://bucketname'... This now includes: copying path '/nix/store/lxnp9cs4cfh2g9r2bs4z7gwwz9kdj2r9-test-package-c' to 's3://bucketname?endpoint=http://server:9000®ion=eu-west-1'... --- .../include/nix/store/s3-binary-cache-store.hh | 2 ++ src/libstore/s3-binary-cache-store.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index e5fcbeda3..288ca41a0 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -75,6 +75,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig static StringSet uriSchemes(); static std::string doc(); + + std::string getHumanReadableURI() const override; }; } // namespace nix diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac08a4982..0b37ac5d7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -29,6 +29,19 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( } } +std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const +{ + auto reference = getReference(); + reference.params = [&]() { + Params relevantParams; + for (auto & setting : s3UriSettings) + if (setting->overridden) + relevantParams.insert({setting->name, reference.params.at(setting->name)}); + return relevantParams; + }(); + return reference.render(); +} + std::string S3BinaryCacheStoreConfig::doc() { return R"( From 459f9e01851cadb7f642cbce473ed7dff215f314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 22 Oct 2025 13:38:05 -0500 Subject: [PATCH 003/155] Fix misleading error messages for missing NARs due to stale cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Nix's SQLite narinfo cache indicates a NAR exists, but the NAR has been garbage collected from the binary cache, Nix displays error messages even though the operation succeeds via fallback. This is misleading because the cached narinfo is simply outdated. This changes SubstituteGone exceptions to produce warnings instead of errors, accurately reflecting that this is an expected cache coherency issue, not an actual failure. Fixes #11411 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/libstore/build/substitution-goal.cc | 8 +++++--- tests/functional/binary-cache.sh | 8 +++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16e530a4..ac18de304 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -268,16 +268,18 @@ Goal::Co PathSubstitutionGoal::tryToRun( try { promise.get_future().get(); } catch (std::exception & e) { - printError(e.what()); - /* Cause the parent build to fail unless --fallback is given, or the substitute has disappeared. The latter case behaves the same as the substitute never having existed in the first place. */ try { throw; - } catch (SubstituteGone &) { + } catch (SubstituteGone & sg) { + /* Missing NARs are expected when they've been garbage collected. + This is not a failure, so log as a warning instead of an error. */ + logWarning({.msg = sg.info().msg}); } catch (...) { + printError(e.what()); substituterFailed = true; } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 2c102df07..d801ac6aa 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -111,7 +111,13 @@ clearStore mv "$cacheDir/nar" "$cacheDir/nar2" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" + +# Verify that missing NARs produce warnings, not errors +# The build should succeed despite the warnings +grepQuiet "does not exist in binary cache" "$TEST_ROOT/log" +# Ensure the message is not at error level by checking that the command succeeded +[ -e "$TEST_ROOT/result" ] mv "$cacheDir/nar2" "$cacheDir/nar" From ad5c6a53b91b6d9b0165e0cf09d9397dfd4657d7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Oct 2025 02:01:14 +0300 Subject: [PATCH 004/155] ci: Move magic-nix-cache-action into install-nix-action composite This reduces duplication and pins the underlying version of magic-nix-cache, as we already do with other actions. --- .github/actions/install-nix-action/action.yaml | 11 +++++++++++ .github/workflows/ci.yml | 6 +----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 46abea179..d694b8eae 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -23,6 +23,10 @@ inputs: github_token: description: "Github token" required: true + use_cache: + description: "Whether to setup magic-nix-cache" + default: true + required: false runs: using: "composite" steps: @@ -118,3 +122,10 @@ runs: source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} extra-conf: ${{ inputs.extra_nix_config }} + - uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13 + if: ${{ inputs.use_cache == 'true' }} + with: + diagnostic-endpoint: '' + use-flakehub: false + use-gha-cache: true + source-revision: c2f46a0afa5f95fd4c184a533afd280c68cf63ff # v0.1.6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1edfcf167..5766ba3c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} + use_cache: false - run: nix flake show --all-systems --json pre-commit-checks: @@ -41,7 +42,6 @@ jobs: dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: DeterminateSystems/magic-nix-cache-action@main - run: ./ci/gha/tests/pre-commit-checks basic-checks: @@ -92,7 +92,6 @@ jobs: dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -230,7 +229,6 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -289,7 +287,6 @@ jobs: extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh profile_build: @@ -310,7 +307,6 @@ jobs: extra_nix_config: | experimental-features = flakes nix-command ca-derivations impure-derivations max-jobs = 1 - - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md cat build-time-report.md >> $GITHUB_STEP_SUMMARY From c8a15bf70d8ec73b11440a6272e20bc23ce10bb3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Oct 2025 02:03:23 +0300 Subject: [PATCH 005/155] ci: Pin cachix action --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5766ba3c1..48fdf09f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@v31 + - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} From f3d8d1f719156ad27a03d2b8211e4295d6f778bf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Oct 2025 02:06:34 +0300 Subject: [PATCH 006/155] ci: Reuse composite install-nix-action for docker_push_image job --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48fdf09f1..8a0820903 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,11 +226,13 @@ jobs: - uses: actions/checkout@v5 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v31 + - uses: ./.github/actions/install-nix-action with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L + dogfood: false + extra_nix_config: | + experimental-features = flakes nix-command + - run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV + - run: nix build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master From 3c83856494e482e31959039fb328557d601fca77 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Oct 2025 02:08:54 +0300 Subject: [PATCH 007/155] ci: Update pinned install_url 2.30.2 -> 2.32.1 --- .github/actions/install-nix-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index d694b8eae..3f668864b 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -16,7 +16,7 @@ inputs: install_url: description: "URL of the Nix installer" required: false - default: "https://releases.nixos.org/nix/nix-2.30.2/install" + default: "https://releases.nixos.org/nix/nix-2.32.1/install" tarball_url: description: "URL of the Nix tarball to use with the experimental installer" required: false From 953929f8990b6ca639b65d9f105c823c770bf7c1 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 20:08:33 +0000 Subject: [PATCH 008/155] fix(libstore): use CURLOPT_POSTFIELDSIZE_LARGE for POST requests Fix POST requests with data to use the correct curl option for specifying body size. Previously used CURLOPT_INFILESIZE_LARGE for both POST and PUT, but POST requires CURLOPT_POSTFIELDSIZE_LARGE. This caused POST request bodies to not be sent correctly, manifesting as S3 multipart CompleteMultipartUpload requests failing with "You must specify at least one part" even though the XML body contained valid parts. --- src/libstore/filetransfer.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 201f2984e..68c8a9e3a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -388,13 +388,15 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_NOBODY, 1); if (request.data) { - if (request.post) + if (request.post) { curl_easy_setopt(req, CURLOPT_POST, 1L); - else + curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length()); + } else { curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this); - curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper); curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } From f594a8e11e06b9ed6b2d62efbf74e964e9c1848c Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Fri, 24 Oct 2025 01:24:04 +0800 Subject: [PATCH 009/155] libexpr needs boost-1.87+ for try_emplace_and_cvisit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 2.32, nix now needs boost 1.87 or later to build, due to using unordered::concurrent_flat_map try_emplace_and_cvisit ../src/libexpr/eval.cc: In member function ‘void nix::EvalState::evalFile(const nix::SourcePath&, nix::Value&, bool)’: ../src/libexpr/eval.cc:1096:20: error: ‘class boost::unordered::concurrent_flat_map, std::equal_to, traceable_allocator > >’ has no member named ‘try_emplace_and_cvisit’; did you mean ‘try_emplace_or_cvisit’? 1096 | fileEvalCache->try_emplace_and_cvisit( | ^~~~~~~~~~~~~~~~~~~~~~ | try_emplace_or_cvisit See https://github.com/boostorg/unordered/commit/834580b53948eec553c232dda40beefc68b3e8f9 --- src/libutil/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index acba0b81b..8b7a5d977 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -64,7 +64,7 @@ boost = dependency( 'url', ], include_type : 'system', - version : '>=1.82.0', + version : '>=1.87.0', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. From c87f29a0b6c575f914b9a687e04f2727ccb5b9b2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Oct 2025 14:03:21 -0400 Subject: [PATCH 010/155] Fix some characterization tests A few changes had cropped up with `_NIX_TEST_ACCEPT=1`: 1. Blake hashing test JSON had a different indentation 2. Store URI had improper non-quoted spaces (1) was is just fixed, as we trust nlohmann JSON to parse JSON correctly, regardless of whitespace. For (2), the existing URL was made a read-only test, since we very much wish to continue parsing such invalid URLs directly. And then the original read/write test was updated to properly percent-encode the space, as the normal form should be. --- src/libstore-tests/data/store-reference/local_3.txt | 2 +- .../data/store-reference/local_3_no_percent.txt | 1 + src/libstore-tests/store-reference.cc | 5 ++++- src/libutil-tests/data/hash/blake3-base64.json | 6 +++--- src/libutil-tests/data/hash/sha256-base64.json | 6 +++--- 5 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 src/libstore-tests/data/store-reference/local_3_no_percent.txt diff --git a/src/libstore-tests/data/store-reference/local_3.txt b/src/libstore-tests/data/store-reference/local_3.txt index 2a67a3426..cd015d74f 100644 --- a/src/libstore-tests/data/store-reference/local_3.txt +++ b/src/libstore-tests/data/store-reference/local_3.txt @@ -1 +1 @@ -local://?root=/foo bar/baz \ No newline at end of file +local://?root=/foo%20bar/baz \ No newline at end of file diff --git a/src/libstore-tests/data/store-reference/local_3_no_percent.txt b/src/libstore-tests/data/store-reference/local_3_no_percent.txt new file mode 100644 index 000000000..2a67a3426 --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_3_no_percent.txt @@ -0,0 +1 @@ +local://?root=/foo bar/baz \ No newline at end of file diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 7b42b45a2..7ae944348 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -100,9 +100,12 @@ URI_TEST(local_1, localExample_1) URI_TEST(local_2, localExample_2) -/* Test path with spaces */ +/* Test path with encoded spaces */ URI_TEST(local_3, localExample_3) +/* Test path with spaces that are improperly not encoded */ +URI_TEST_READ(local_3_no_percent, localExample_3) + URI_TEST_READ(local_shorthand_1, localExample_1) URI_TEST_READ(local_shorthand_2, localExample_2) diff --git a/src/libutil-tests/data/hash/blake3-base64.json b/src/libutil-tests/data/hash/blake3-base64.json index d668c2d9c..b9a20cdb4 100644 --- a/src/libutil-tests/data/hash/blake3-base64.json +++ b/src/libutil-tests/data/hash/blake3-base64.json @@ -1,5 +1,5 @@ { - "algorithm": "blake3", - "format": "base64", - "hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA=" + "algorithm": "blake3", + "format": "base64", + "hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA=" } diff --git a/src/libutil-tests/data/hash/sha256-base64.json b/src/libutil-tests/data/hash/sha256-base64.json index 239764dd1..838af80a7 100644 --- a/src/libutil-tests/data/hash/sha256-base64.json +++ b/src/libutil-tests/data/hash/sha256-base64.json @@ -1,5 +1,5 @@ { - "algorithm": "sha256", - "format": "base64", - "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts=" + "algorithm": "sha256", + "format": "base64", + "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts=" } From d6f1e2de21b090ada8f658751ec1c528f13316bc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Oct 2025 21:17:09 +0200 Subject: [PATCH 011/155] Merge pull request #14323 from NixOS/skip-nar-parse addToStore(): Don't parse the NAR * StringSource: Implement skip() This is slightly faster than doing a read() into a buffer just to discard the data. * LocalStore::addToStore(): Skip unnecessary NARs rather than parsing them Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/libstore/local-store.cc | 6 ++---- src/libutil/include/nix/util/serialise.hh | 2 ++ src/libutil/serialise.cc | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 59d5cc24f..3f108f97e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1048,15 +1048,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF /* In case we are not interested in reading the NAR: discard it. */ bool narRead = false; Finally cleanup = [&]() { - if (!narRead) { - NullFileSystemObjectSink sink; + if (!narRead) try { - parseDump(sink, source); + source.skip(info.narSize); } catch (...) { // TODO: should Interrupted be handled here? ignoreExceptionInDestructor(); } - } }; addTempRoot(info.path); diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 8799e128f..d6845a494 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -255,6 +255,8 @@ struct StringSource : Source } size_t read(char * data, size_t len) override; + + void skip(size_t len) override; }; /** diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 47a00c8d6..ba153625e 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -242,6 +242,16 @@ size_t StringSource::read(char * data, size_t len) return n; } +void StringSource::skip(size_t len) +{ + const size_t remain = s.size() - pos; + if (len > remain) { + pos = s.size(); + throw EndOfFile("end of string reached"); + } + pos += len; +} + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink From 4f5af471fbfc0f551ace552b3f8f184641814313 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Oct 2025 23:49:41 +0300 Subject: [PATCH 012/155] Revert "libmain: Catch logger exceptions in `handleExceptions`" This reverts commit 90d1ff480590b56db202a20c3927df4bf05e4eac. The initial issue with EPIPE was solved in 9f680874c5aa15304c3ab3b942170a743287f87b. Now this patch does move bad than good by eating up boost::io::format_error that are bugs. --- src/libmain/shared.cc | 49 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7187e9720..4b36ec98e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -320,34 +320,29 @@ int handleExceptions(const std::string & programName, std::function fun) std::string error = ANSI_RED "error:" ANSI_NORMAL " "; try { try { - try { - fun(); - } catch (...) { - /* Subtle: we have to make sure that any `interrupted' - condition is discharged before we reach printMsg() - below, since otherwise it will throw an (uncaught) - exception. */ - setInterruptThrown(); - throw; - } - } catch (Exit & e) { - return e.status; - } catch (UsageError & e) { - logError(e.info()); - printError("Try '%1% --help' for more information.", programName); - return 1; - } catch (BaseError & e) { - logError(e.info()); - return e.info().status; - } catch (std::bad_alloc & e) { - printError(error + "out of memory"); - return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + fun(); + } catch (...) { + /* Subtle: we have to make sure that any `interrupted' + condition is discharged before we reach printMsg() + below, since otherwise it will throw an (uncaught) + exception. */ + setInterruptThrown(); + throw; } - } catch (...) { - /* In case logger also throws just give up. */ + } catch (Exit & e) { + return e.status; + } catch (UsageError & e) { + logError(e.info()); + printError("Try '%1% --help' for more information.", programName); + return 1; + } catch (BaseError & e) { + logError(e.info()); + return e.info().status; + } catch (std::bad_alloc & e) { + printError(error + "out of memory"); + return 1; + } catch (std::exception & e) { + printError(error + e.what()); return 1; } From b5ae3e10c27a1fd0b3de453dcf02c83e3f2c4e10 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 24 Oct 2025 00:29:08 +0300 Subject: [PATCH 013/155] libstore/filetransfer: Remove verifyTLS from FileTransferRequest, since it's always true This variable is always true, so there's no use-case for it anymore. --- src/libstore/filetransfer.cc | 9 ++------- src/libstore/include/nix/store/filetransfer.hh | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 201f2984e..1c97cf400 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -399,13 +399,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } - if (request.verifyTLS) { - if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); - } else { - curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); - } + if (settings.caFile != "") + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000 curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 34ec316ef..2b86f6ac9 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -99,7 +99,6 @@ struct FileTransferRequest VerbatimURL uri; Headers headers; std::string expectedETag; - bool verifyTLS = true; bool head = false; bool post = false; size_t tries = fileTransferSettings.tries; From 4c4eb5d07fc59513fb44ca348b42a33015a65e50 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 24 Oct 2025 01:34:06 +0300 Subject: [PATCH 014/155] ci: Bump magic-nix-cache with post-build-hook fix No tagged release with the fix for [^]. [^]: https://github.com/DeterminateSystems/magic-nix-cache/commit/578f01e1473129cc289d579ba6dee9cdac40aeab --- .github/actions/install-nix-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 3f668864b..00d02d6a2 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -128,4 +128,4 @@ runs: diagnostic-endpoint: '' use-flakehub: false use-gha-cache: true - source-revision: c2f46a0afa5f95fd4c184a533afd280c68cf63ff # v0.1.6 + source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main From 7308fde0bc0ee9a902891385453ef6313cb91194 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Oct 2025 19:18:57 +0200 Subject: [PATCH 015/155] Allow access to the result of fetchClosure --- src/libexpr/primops/fetchClosure.cc | 6 ++++++ tests/functional/dependencies.builder0.sh | 2 ++ tests/functional/fetchClosure.sh | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 63da53aa9..6e1389814 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -64,6 +64,8 @@ static void runFetchClosureWithRewrite( .pos = state.positions[pos]}); } + state.allowClosure(toPath); + state.mkStorePathString(toPath, v); } @@ -91,6 +93,8 @@ static void runFetchClosureWithContentAddressedPath( .pos = state.positions[pos]}); } + state.allowClosure(fromPath); + state.mkStorePathString(fromPath, v); } @@ -115,6 +119,8 @@ static void runFetchClosureWithInputAddressedPath( .pos = state.positions[pos]}); } + state.allowClosure(fromPath); + state.mkStorePathString(fromPath, v); } diff --git a/tests/functional/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh index 6fbe4a07a..f680cf7f2 100644 --- a/tests/functional/dependencies.builder0.sh +++ b/tests/functional/dependencies.builder0.sh @@ -17,4 +17,6 @@ ln -s "$out" "$out"/self echo program > "$out"/program chmod +x "$out"/program +echo '1 + 2' > "$out"/foo.nix + echo FOO diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh index 9b79ab396..85a83d192 100755 --- a/tests/functional/fetchClosure.sh +++ b/tests/functional/fetchClosure.sh @@ -99,6 +99,14 @@ clearStore [ -e "$caPath" ] +# Test import-from-derivation on the result of fetchClosure. +[[ $(nix eval -v --expr " + import \"\${builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + }}/foo.nix\" +") = 3 ]] + # Check that URL query parameters aren't allowed. clearStore narCache=$TEST_ROOT/nar-cache From 8d338c9234b8e7758eb8683174741be5d350d1aa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Oct 2025 17:11:37 -0400 Subject: [PATCH 016/155] JSON Schema for `DerivedPath` Note that this is "deriving path" in the manual -- the great sed of the code base to bring it in sync has yet to happen yet. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/deriving-path.md | 21 +++++++++++++++ doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/deriving-path-v1 | 1 + .../json/schema/deriving-path-v1.yaml | 27 +++++++++++++++++++ src/json-schema-checks/deriving-path | 1 + src/json-schema-checks/meson.build | 9 +++++++ src/json-schema-checks/package.nix | 1 + 9 files changed, 63 insertions(+) create mode 100644 doc/manual/source/protocols/json/deriving-path.md create mode 120000 doc/manual/source/protocols/json/schema/deriving-path-v1 create mode 100644 doc/manual/source/protocols/json/schema/deriving-path-v1.yaml create mode 120000 src/json-schema-checks/deriving-path diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 30486869e..eb20f8714 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -35,6 +35,7 @@ mkMesonDerivation (finalAttrs: { ../../.version # For example JSON ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/derived-path # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index f74ed7043..b4796f652 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -120,6 +120,7 @@ - [Hash](protocols/json/hash.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) + - [Deriving Path](protocols/json/deriving-path.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive.md) diff --git a/doc/manual/source/protocols/json/deriving-path.md b/doc/manual/source/protocols/json/deriving-path.md new file mode 100644 index 000000000..9851b371d --- /dev/null +++ b/doc/manual/source/protocols/json/deriving-path.md @@ -0,0 +1,21 @@ +{{#include deriving-path-v1-fixed.md}} + +## Examples + +### Constant + +```json +{{#include schema/deriving-path-v1/single_opaque.json}} +``` + +### Output of static derivation + +```json +{{#include schema/deriving-path-v1/single_built.json}} +``` + +### Output of dynamic derivation + +```json +{{#include schema/deriving-path-v1/single_built_built.json}} +``` diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 44795599c..191ec6dbe 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -11,6 +11,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', 'derivation-v3', + 'deriving-path-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1 b/doc/manual/source/protocols/json/schema/deriving-path-v1 new file mode 120000 index 000000000..92ec6d01a --- /dev/null +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/derived-path \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml new file mode 100644 index 000000000..9c0350d3d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml @@ -0,0 +1,27 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json +title: Deriving Path +description: | + This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path). +oneOf: + - title: Constant + description: | + See [Constant](@docroot@/store/derivation/index.md#deriving-path-constant) deriving path. + type: string + - title: Output + description: | + See [Output](@docroot@/store/derivation/index.md#deriving-path-output) deriving path. + type: object + properties: + drvPath: + "$ref": "#" + description: | + A deriving path to a [Derivation](@docroot@/store/derivation/index.md#store-derivation), whose output is being referred to. + output: + type: string + description: | + The name of an output produced by that derivation (e.g. "out", "doc", etc.). + required: + - drvPath + - output + additionalProperties: false diff --git a/src/json-schema-checks/deriving-path b/src/json-schema-checks/deriving-path new file mode 120000 index 000000000..4f50b2ee9 --- /dev/null +++ b/src/json-schema-checks/deriving-path @@ -0,0 +1 @@ +../../src/libstore-tests/data/derived-path \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index ebd6f6b2b..09da8770b 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -52,6 +52,15 @@ schemas = [ # 'output-inputAddressed.json', # ], # }, + { + 'stem' : 'deriving-path', + 'schema' : schema_dir / 'deriving-path-v1.yaml', + 'files' : [ + 'single_opaque.json', + 'single_built.json', + 'single_built_built.json', + ], + }, ] # Validate each example against the schema diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 41458adb8..cf4e4cb19 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -22,6 +22,7 @@ mkMesonDerivation (finalAttrs: { ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/derivation + ../../src/libstore-tests/data/derived-path ./. ]; From f1968ea38e51201b37962a9cfd80775989a56d46 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 23 Oct 2025 05:03:16 +0000 Subject: [PATCH 017/155] refactor(libstore): replace HTTP method boolean flags with enum Replace the individual boolean flags (head, post) with a unified HttpMethod enum struct in FileTransferRequest. --- src/libfetchers/git-lfs-fetch.cc | 2 +- src/libstore/filetransfer.cc | 4 ++-- src/libstore/http-binary-cache-store.cc | 2 +- .../include/nix/store/filetransfer.hh | 21 ++++++++++++++++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 9688daa4a..aee1163bb 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -209,7 +209,7 @@ std::vector Fetch::fetchUrls(const std::vector & pointe auto url = api.endpoint + "/objects/batch"; const auto & authHeader = api.authHeader; FileTransferRequest request(parseURL(url)); - request.post = true; + request.method = HttpMethod::POST; Headers headers; if (authHeader.has_value()) headers.push_back({"Authorization", *authHeader}); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b9e52e7d5..c2c2a86c4 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -384,11 +384,11 @@ struct curlFileTransfer : public FileTransfer if (settings.downloadSpeed.get() > 0) curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); - if (request.head) + if (request.method == HttpMethod::HEAD) curl_easy_setopt(req, CURLOPT_NOBODY, 1); if (request.data) { - if (request.post) { + if (request.method == HttpMethod::POST) { curl_easy_setopt(req, CURLOPT_POST, 1L); curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length()); } else { diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9567aec2f..5c455dd04 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -139,7 +139,7 @@ protected: try { FileTransferRequest request(makeRequest(path)); - request.head = true; + request.method = HttpMethod::HEAD; getFileTransfer()->download(request); return true; } catch (FileTransferError & e) { diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 2b86f6ac9..286014517 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -83,6 +83,15 @@ extern FileTransferSettings fileTransferSettings; extern const unsigned int RETRY_TIME_MS_DEFAULT; +/** + * HTTP methods supported by FileTransfer. + */ +enum struct HttpMethod { + GET, + HEAD, + POST, +}; + /** * Username and optional password for HTTP basic authentication. * These are used with curl's CURLOPT_USERNAME and CURLOPT_PASSWORD options @@ -99,8 +108,7 @@ struct FileTransferRequest VerbatimURL uri; Headers headers; std::string expectedETag; - bool head = false; - bool post = false; + HttpMethod method = HttpMethod::GET; size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; @@ -129,7 +137,14 @@ struct FileTransferRequest std::string verb() const { - return data ? "upload" : "download"; + switch (method) { + case HttpMethod::HEAD: + case HttpMethod::GET: + return "download"; + case HttpMethod::POST: + return "upload"; + } + unreachable(); } private: From d924374bf22176263409c3ad49982aad4e906b2f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 23 Oct 2025 20:48:21 +0000 Subject: [PATCH 018/155] docs(libstore): document verb() method returns verb root for gerund form Add documentation to FileTransferRequest::verb() explaining that it returns a verb root intended to be concatenated with "ing" to form the gerund. --- src/libstore/include/nix/store/filetransfer.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 286014517..a9ed05dc8 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -135,6 +135,11 @@ struct FileTransferRequest { } + /** + * Returns the verb root for logging purposes. + * The returned string is intended to be concatenated with "ing" to form the gerund, + * e.g., "download" + "ing" -> "downloading", "upload" + "ing" -> "uploading". + */ std::string verb() const { switch (method) { From afe5ed879f7015e62bcf431f0bdf70093af63ca5 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 21 Oct 2025 09:02:36 +0000 Subject: [PATCH 019/155] feat(libstore): add DELETE method support to FileTransfer Add support for HTTP DELETE requests to FileTransfer infrastructure: This enables S3 multipart upload abort functionality via DELETE requests to S3 endpoints. --- src/libstore/filetransfer.cc | 8 ++++++++ src/libstore/include/nix/store/filetransfer.hh | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c2c2a86c4..9fd7a967b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -387,6 +387,9 @@ struct curlFileTransfer : public FileTransfer if (request.method == HttpMethod::HEAD) curl_easy_setopt(req, CURLOPT_NOBODY, 1); + if (request.method == HttpMethod::DELETE) + curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (request.data) { if (request.method == HttpMethod::POST) { curl_easy_setopt(req, CURLOPT_POST, 1L); @@ -919,6 +922,11 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request) return enqueueFileTransfer(request).get(); } +FileTransferResult FileTransfer::deleteResource(const FileTransferRequest & request) +{ + return enqueueFileTransfer(request).get(); +} + void FileTransfer::download( FileTransferRequest && request, Sink & sink, std::function resultCallback) { diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index a9ed05dc8..402ee4900 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -90,6 +90,7 @@ enum struct HttpMethod { GET, HEAD, POST, + DELETE, }; /** @@ -148,6 +149,8 @@ struct FileTransferRequest return "download"; case HttpMethod::POST: return "upload"; + case HttpMethod::DELETE: + return "delet"; } unreachable(); } @@ -221,6 +224,11 @@ struct FileTransfer */ FileTransferResult upload(const FileTransferRequest & request); + /** + * Synchronously delete a resource. + */ + FileTransferResult deleteResource(const FileTransferRequest & request); + /** * Download a file, writing its data to a sink. The sink will be * invoked on the thread of the caller. From 6b7223b6b77c1e0c719f4492496fed22d4cc6830 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 21 Oct 2025 06:11:23 +0000 Subject: [PATCH 020/155] refactor(libstore): add sizeHint parameter to upsertFile() Add a sizeHint parameter to BinaryCacheStore::upsertFile() to enable size-based upload decisions in implementations. This lays the groundwork for reintroducing S3 multipart upload support. --- src/libstore/binary-cache-store.cc | 8 +++++--- src/libstore/http-binary-cache-store.cc | 3 ++- .../include/nix/store/binary-cache-store.hh | 18 ++++++++++++++++-- src/libstore/local-binary-cache-store.cc | 3 ++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3705f3d4d..9bb81add7 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -76,9 +76,10 @@ std::optional BinaryCacheStore::getNixCacheInfo() return getFile(cacheInfoFile); } -void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType) +void BinaryCacheStore::upsertFile( + const std::string & path, std::string && data, const std::string & mimeType, uint64_t sizeHint) { - upsertFile(path, std::make_shared(std::move(data)), mimeType); + upsertFile(path, std::make_shared(std::move(data)), mimeType, sizeHint); } void BinaryCacheStore::getFile(const std::string & path, Callback> callback) noexcept @@ -274,7 +275,8 @@ ref BinaryCacheStore::addToStoreCommon( upsertFile( narInfo->url, std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary), - "application/x-nix-nar"); + "application/x-nix-nar", + narInfo->fileSize); } else stats.narWriteAverted++; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8d5f427af..c4dcf0124 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -143,7 +143,8 @@ protected: void upsertFile( const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) override + const std::string & mimeType, + uint64_t sizeHint) override { auto req = makeRequest(path); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 3f4de2bd4..0bed09aec 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -101,13 +101,27 @@ public: virtual bool fileExists(const std::string & path) = 0; virtual void upsertFile( - const std::string & path, std::shared_ptr> istream, const std::string & mimeType) = 0; + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType, + uint64_t sizeHint) = 0; void upsertFile( const std::string & path, // FIXME: use std::string_view std::string && data, - const std::string & mimeType); + const std::string & mimeType, + uint64_t sizeHint); + + void upsertFile( + const std::string & path, + // FIXME: use std::string_view + std::string && data, + const std::string & mimeType) + { + auto size = data.size(); + upsertFile(path, std::move(data), mimeType, size); + } /** * Dump the contents of the specified file to a sink. diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b5e43de68..c1811bf17 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -56,7 +56,8 @@ protected: void upsertFile( const std::string & path, std::shared_ptr> istream, - const std::string & mimeType) override + const std::string & mimeType, + uint64_t sizeHint) override { auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; From 476c21d5ef711397b7c234554bc292c6162d3764 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 21 Oct 2025 08:45:10 +0000 Subject: [PATCH 021/155] refactor(libstore): expose HttpBinaryCacheStore and add S3BinaryCacheStore Move HttpBinaryCacheStore class from .cc file to header to enable inheritance by S3BinaryCacheStore. Create S3BinaryCacheStore class that overrides upsertFile() to implement multipart upload logic. --- src/libstore/http-binary-cache-store.cc | 406 +++++++++--------- .../nix/store/http-binary-cache-store.hh | 51 +++ .../nix/store/s3-binary-cache-store.hh | 2 + src/libstore/s3-binary-cache-store.cc | 37 ++ 4 files changed, 284 insertions(+), 212 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 7883161d5..945fe1834 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -51,227 +51,209 @@ std::string HttpBinaryCacheStoreConfig::doc() ; } -class HttpBinaryCacheStore : public virtual BinaryCacheStore +HttpBinaryCacheStore::HttpBinaryCacheStore(ref config) + : Store{*config} // TODO it will actually mutate the configuration + , BinaryCacheStore{*config} + , config{config} { - struct State - { - bool enabled = true; - std::chrono::steady_clock::time_point disabledUntil; - }; + diskCache = getNarInfoDiskCache(); +} - Sync _state; - -public: - - using Config = HttpBinaryCacheStoreConfig; - - ref config; - - HttpBinaryCacheStore(ref config) - : Store{*config} // TODO it will actually mutate the configuration - , BinaryCacheStore{*config} - , config{config} - { - diskCache = getNarInfoDiskCache(); - } - - void init() override - { - // FIXME: do this lazily? - // For consistent cache key handling, use the reference without parameters - // This matches what's used in Store::queryPathInfo() lookups - auto cacheKey = config->getReference().render(/*withParams=*/false); - - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); - } else { - try { - BinaryCacheStore::init(); - } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); - } - diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); - } - } - -protected: - - std::optional getCompressionMethod(const std::string & path) - { - if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty()) - return config->narinfoCompression; - else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty()) - return config->lsCompression; - else if (hasPrefix(path, "log/") && !config->logCompression.get().empty()) - return config->logCompression; - else - return std::nullopt; - } - - void maybeDisable() - { - auto state(_state.lock()); - if (state->enabled && settings.tryFallback) { - int t = 60; - printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t); - state->enabled = false; - state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); - } - } - - void checkEnabled() - { - auto state(_state.lock()); - if (state->enabled) - return; - if (std::chrono::steady_clock::now() > state->disabledUntil) { - state->enabled = true; - debug("re-enabling binary cache '%s'", config->getHumanReadableURI()); - return; - } - throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI()); - } - - bool fileExists(const std::string & path) override - { - checkEnabled(); +void HttpBinaryCacheStore::init() +{ + // FIXME: do this lazily? + // For consistent cache key handling, use the reference without parameters + // This matches what's used in Store::queryPathInfo() lookups + auto cacheKey = config->getReference().render(/*withParams=*/false); + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { + config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config->priority.setDefault(cacheInfo->priority); + } else { try { - FileTransferRequest request(makeRequest(path)); - request.method = HttpMethod::HEAD; - getFileTransfer()->download(request); - return true; - } catch (FileTransferError & e) { - /* S3 buckets return 403 if a file doesn't exist and the - bucket is unlistable, so treat 403 as 404. */ - if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - return false; - maybeDisable(); - throw; + BinaryCacheStore::init(); + } catch (UploadToHTTP &) { + throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } + diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); } +} - void upsertFile( - const std::string & path, - std::shared_ptr> istream, - const std::string & mimeType, - uint64_t sizeHint) override - { - auto req = makeRequest(path); - - auto data = StreamToSourceAdapter(istream).drain(); - - if (auto compressionMethod = getCompressionMethod(path)) { - data = compress(*compressionMethod, data); - req.headers.emplace_back("Content-Encoding", *compressionMethod); - } - - req.data = std::move(data); - req.mimeType = mimeType; - - try { - getFileTransfer()->upload(req); - } catch (FileTransferError & e) { - throw UploadToHTTP( - "while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg()); - } - } - - FileTransferRequest makeRequest(const std::string & path) - { - /* Otherwise the last path fragment will get discarded. */ - auto cacheUriWithTrailingSlash = config->cacheUri; - if (!cacheUriWithTrailingSlash.path.empty()) - cacheUriWithTrailingSlash.path.push_back(""); - - /* path is not a path, but a full relative or absolute - URL, e.g. we've seen in the wild NARINFO files have a URL - field which is - `nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj` - (note the query param) and that gets passed here. */ - auto result = parseURLRelative(path, cacheUriWithTrailingSlash); - - /* For S3 URLs, preserve query parameters from the base URL when the - relative path doesn't have its own query parameters. This is needed - to preserve S3-specific parameters like endpoint and region. */ - if (config->cacheUri.scheme == "s3" && result.query.empty()) { - result.query = config->cacheUri.query; - } - - return FileTransferRequest(result); - } - - void getFile(const std::string & path, Sink & sink) override - { - checkEnabled(); - auto request(makeRequest(path)); - try { - getFileTransfer()->download(std::move(request), sink); - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - throw NoSuchBinaryCacheFile( - "file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI()); - maybeDisable(); - throw; - } - } - - void getFile(const std::string & path, Callback> callback) noexcept override - { - auto callbackPtr = std::make_shared(std::move(callback)); - - try { - checkEnabled(); - - auto request(makeRequest(path)); - - getFileTransfer()->enqueueFileTransfer( - request, {[callbackPtr, this](std::future result) { - try { - (*callbackPtr)(std::move(result.get().data)); - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - return (*callbackPtr)({}); - maybeDisable(); - callbackPtr->rethrow(); - } catch (...) { - callbackPtr->rethrow(); - } - }}); - - } catch (...) { - callbackPtr->rethrow(); - return; - } - } - - std::optional getNixCacheInfo() override - { - try { - auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); - return result.data; - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound) - return std::nullopt; - maybeDisable(); - throw; - } - } - - /** - * This isn't actually necessary read only. We support "upsert" now, so we - * have a notion of authentication via HTTP POST/PUT. - * - * For now, we conservatively say we don't know. - * - * \todo try to expose our HTTP authentication status. - */ - std::optional isTrustedClient() override - { +std::optional HttpBinaryCacheStore::getCompressionMethod(const std::string & path) +{ + if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty()) + return config->narinfoCompression; + else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty()) + return config->lsCompression; + else if (hasPrefix(path, "log/") && !config->logCompression.get().empty()) + return config->logCompression; + else return std::nullopt; +} + +void HttpBinaryCacheStore::maybeDisable() +{ + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); } -}; +} + +void HttpBinaryCacheStore::checkEnabled() +{ + auto state(_state.lock()); + if (state->enabled) + return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", config->getHumanReadableURI()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI()); +} + +bool HttpBinaryCacheStore::fileExists(const std::string & path) +{ + checkEnabled(); + + try { + FileTransferRequest request(makeRequest(path)); + request.method = HttpMethod::HEAD; + getFileTransfer()->download(request); + return true; + } catch (FileTransferError & e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + return false; + maybeDisable(); + throw; + } +} + +void HttpBinaryCacheStore::upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType, + uint64_t sizeHint) +{ + auto req = makeRequest(path); + + auto data = StreamToSourceAdapter(istream).drain(); + + auto compressionMethod = getCompressionMethod(path); + + if (compressionMethod) { + data = compress(*compressionMethod, data); + req.headers.emplace_back("Content-Encoding", *compressionMethod); + } + + req.data = std::move(data); + req.mimeType = mimeType; + + try { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg()); + } +} + +FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path) +{ + /* Otherwise the last path fragment will get discarded. */ + auto cacheUriWithTrailingSlash = config->cacheUri; + if (!cacheUriWithTrailingSlash.path.empty()) + cacheUriWithTrailingSlash.path.push_back(""); + + /* path is not a path, but a full relative or absolute + URL, e.g. we've seen in the wild NARINFO files have a URL + field which is + `nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj` + (note the query param) and that gets passed here. */ + auto result = parseURLRelative(path, cacheUriWithTrailingSlash); + + /* For S3 URLs, preserve query parameters from the base URL when the + relative path doesn't have its own query parameters. This is needed + to preserve S3-specific parameters like endpoint and region. */ + if (config->cacheUri.scheme == "s3" && result.query.empty()) { + result.query = config->cacheUri.query; + } + + return FileTransferRequest(result); +} + +void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink) +{ + checkEnabled(); + auto request(makeRequest(path)); + try { + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI()); + maybeDisable(); + throw; + } +} + +void HttpBinaryCacheStore::getFile(const std::string & path, Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + try { + checkEnabled(); + + auto request(makeRequest(path)); + + getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future result) { + try { + (*callbackPtr)(std::move(result.get().data)); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound + || e.error == FileTransfer::Forbidden) + return (*callbackPtr)({}); + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + + } catch (...) { + callbackPtr->rethrow(); + return; + } +} + +std::optional HttpBinaryCacheStore::getNixCacheInfo() +{ + try { + auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + return result.data; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) + return std::nullopt; + maybeDisable(); + throw; + } +} + +/** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ +std::optional HttpBinaryCacheStore::isTrustedClient() +{ + return std::nullopt; +} ref HttpBinaryCacheStore::Config::openStore() const { diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index e0b7ac1ea..d8ba72390 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -3,6 +3,10 @@ #include "nix/util/url.hh" #include "nix/store/binary-cache-store.hh" +#include "nix/store/filetransfer.hh" +#include "nix/util/sync.hh" + +#include namespace nix { @@ -46,4 +50,51 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this _state; + +public: + + using Config = HttpBinaryCacheStoreConfig; + + ref config; + + HttpBinaryCacheStore(ref config); + + void init() override; + +protected: + + std::optional getCompressionMethod(const std::string & path); + + void maybeDisable(); + + void checkEnabled(); + + bool fileExists(const std::string & path) override; + + void upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType, + uint64_t sizeHint) override; + + FileTransferRequest makeRequest(const std::string & path); + + void getFile(const std::string & path, Sink & sink) override; + + void getFile(const std::string & path, Callback> callback) noexcept override; + + std::optional getNixCacheInfo() override; + + std::optional isTrustedClient() override; +}; + } // namespace nix diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 288ca41a0..81a2d3f3f 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -77,6 +77,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig static std::string doc(); std::string getHumanReadableURI() const override; + + ref openStore() const override; }; } // namespace nix diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 0b37ac5d7..5d97fb0fd 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,36 @@ namespace nix { +class S3BinaryCacheStore : public virtual HttpBinaryCacheStore +{ +public: + S3BinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , HttpBinaryCacheStore{config} + , s3Config{config} + { + } + + void upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType, + uint64_t sizeHint) override; + +private: + ref s3Config; +}; + +void S3BinaryCacheStore::upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType, + uint64_t sizeHint) +{ + HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; @@ -51,6 +81,13 @@ std::string S3BinaryCacheStoreConfig::doc() )"; } +ref S3BinaryCacheStoreConfig::openStore() const +{ + auto sharedThis = std::const_pointer_cast( + std::static_pointer_cast(shared_from_this())); + return make_ref(ref{sharedThis}); +} + static RegisterStoreImplementation registerS3BinaryCacheStore; } // namespace nix From 1cd8458c28bb06c90486023a52db3cee706a7a70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 25 Oct 2025 02:09:06 +0300 Subject: [PATCH 022/155] tests/functional: Add source-paths tests This has already been implemented in 1e709554d565be51ab8d5a7e4941b0cc1da70807 as a side-effect of mounting the accessors in storeFS. Let's test this so it doesn't regress. (cherry-picked from https://github.com/NixOS/nix/pull/12915) --- tests/functional/flakes/source-paths.sh | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/functional/flakes/source-paths.sh b/tests/functional/flakes/source-paths.sh index 4709bf2fc..3aa3683c2 100644 --- a/tests/functional/flakes/source-paths.sh +++ b/tests/functional/flakes/source-paths.sh @@ -12,6 +12,10 @@ cat > "$repo/flake.nix" < "$repo/foo" + +expectStderr 1 nix eval "$repo#z" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." +expectStderr 1 nix eval "$repo#a" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/foo" + +[[ $(nix eval --raw "$repo#z") = 123 ]] + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' does not exist in Git repository \"$repo\"." + +mkdir -p "$repo/dir" +echo 456 > "$repo/dir/default.nix" + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/dir/default.nix" + +[[ $(nix eval "$repo#b") = 456 ]] From 78e98691d64d50baa2c6e6146250c4d87b177751 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:52:32 +0000 Subject: [PATCH 023/155] refactor(libstore/filetransfer): make setupForS3 public --- src/libstore/include/nix/store/filetransfer.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 402ee4900..305c33af1 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -155,9 +155,10 @@ struct FileTransferRequest unreachable(); } + void setupForS3(); + private: friend struct curlFileTransfer; - void setupForS3(); #if NIX_WITH_AWS_AUTH std::optional awsSigV4Provider; #endif From e38128b90d35d31a9ea69802f62219e27f82c12a Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 16 Oct 2025 18:38:42 +0000 Subject: [PATCH 024/155] feat(libstore): support S3 object versioning via versionId parameter S3 buckets support object versioning to prevent unexpected changes, but Nix previously lacked the ability to fetch specific versions of S3 objects. This adds support for a `versionId` query parameter in S3 URLs, enabling users to pin to specific object versions: ``` s3://bucket/key?region=us-east-1&versionId=abc123 ``` --- doc/manual/rl-next/s3-object-versioning.md | 14 ++++++ src/libstore-tests/s3-url.cc | 50 ++++++++++++++++++++ src/libstore/include/nix/store/s3-url.hh | 1 + src/libstore/s3-url.cc | 10 ++++ tests/nixos/s3-binary-cache-store.nix | 53 ++++++++++++++++++++-- 5 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 doc/manual/rl-next/s3-object-versioning.md diff --git a/doc/manual/rl-next/s3-object-versioning.md b/doc/manual/rl-next/s3-object-versioning.md new file mode 100644 index 000000000..3b85e0926 --- /dev/null +++ b/doc/manual/rl-next/s3-object-versioning.md @@ -0,0 +1,14 @@ +--- +synopsis: "S3 URLs now support object versioning via versionId parameter" +prs: [14274] +issues: [13955] +--- + +S3 URLs now support a `versionId` query parameter to fetch specific versions +of objects from S3 buckets with versioning enabled. This allows pinning to +exact object versions for reproducibility and protection against unexpected +changes: + +``` +s3://bucket/key?region=us-east-1&versionId=abc123def456 +``` diff --git a/src/libstore-tests/s3-url.cc b/src/libstore-tests/s3-url.cc index 2c384c255..9fa625fd6 100644 --- a/src/libstore-tests/s3-url.cc +++ b/src/libstore-tests/s3-url.cc @@ -70,6 +70,25 @@ INSTANTIATE_TEST_SUITE_P( }, "with_profile_and_region", }, + ParsedS3URLTestCase{ + "s3://my-bucket/my-key.txt?versionId=abc123xyz", + { + .bucket = "my-bucket", + .key = {"my-key.txt"}, + .versionId = "abc123xyz", + }, + "with_versionId", + }, + ParsedS3URLTestCase{ + "s3://bucket/path/to/object?region=eu-west-1&versionId=version456", + { + .bucket = "bucket", + .key = {"path", "to", "object"}, + .region = "eu-west-1", + .versionId = "version456", + }, + "with_region_and_versionId", + }, ParsedS3URLTestCase{ "s3://bucket/key?endpoint=https://minio.local&scheme=http", { @@ -222,6 +241,37 @@ INSTANTIATE_TEST_SUITE_P( }, "https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt", "complex_path_and_region", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my-bucket", + .key = {"my-key.txt"}, + .versionId = "abc123xyz", + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, + .path = {"", "my-bucket", "my-key.txt"}, + .query = {{"versionId", "abc123xyz"}}, + }, + "https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz", + "with_versionId", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "versioned-bucket", + .key = {"path", "to", "object"}, + .region = "eu-west-1", + .versionId = "version456", + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"}, + .path = {"", "versioned-bucket", "path", "to", "object"}, + .query = {{"versionId", "version456"}}, + }, + "https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456", + "with_region_and_versionId", }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); diff --git a/src/libstore/include/nix/store/s3-url.hh b/src/libstore/include/nix/store/s3-url.hh index 4ee0c87f9..cf59dbea8 100644 --- a/src/libstore/include/nix/store/s3-url.hh +++ b/src/libstore/include/nix/store/s3-url.hh @@ -26,6 +26,7 @@ struct ParsedS3URL std::optional profile; std::optional region; std::optional scheme; + std::optional versionId; /** * The endpoint can be either missing, be an absolute URI (with a scheme like `http:`) * or an authority (so an IP address or a registered name). diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc index e8fbba8f7..503c0cd91 100644 --- a/src/libstore/s3-url.cc +++ b/src/libstore/s3-url.cc @@ -48,6 +48,7 @@ try { .profile = getOptionalParam("profile"), .region = getOptionalParam("region"), .scheme = getOptionalParam("scheme"), + .versionId = getOptionalParam("versionId"), .endpoint = [&]() -> decltype(ParsedS3URL::endpoint) { if (!endpoint) return std::monostate(); @@ -73,6 +74,12 @@ ParsedURL ParsedS3URL::toHttpsUrl() const auto regionStr = region.transform(toView).value_or("us-east-1"); auto schemeStr = scheme.transform(toView).value_or("https"); + // Build query parameters (e.g., versionId if present) + StringMap queryParams; + if (versionId) { + queryParams["versionId"] = *versionId; + } + // Handle endpoint configuration using std::visit return std::visit( overloaded{ @@ -85,6 +92,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const .scheme = std::string{schemeStr}, .authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"}, .path = std::move(path), + .query = std::move(queryParams), }; }, [&](const ParsedURL::Authority & auth) { @@ -96,6 +104,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const .scheme = std::string{schemeStr}, .authority = auth, .path = std::move(path), + .query = std::move(queryParams), }; }, [&](const ParsedURL & endpointUrl) { @@ -107,6 +116,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const .scheme = endpointUrl.scheme, .authority = endpointUrl.authority, .path = std::move(path), + .query = std::move(queryParams), }; }, }, diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 981fab868..a2ede4572 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -1,7 +1,5 @@ { - lib, config, - nixpkgs, ... }: @@ -147,7 +145,7 @@ in else: machine.fail(f"nix path-info {pkg}") - def setup_s3(populate_bucket=[], public=False): + def setup_s3(populate_bucket=[], public=False, versioned=False): """ Decorator that creates/destroys a unique bucket for each test. Optionally pre-populates bucket with specified packages. @@ -156,14 +154,17 @@ in Args: populate_bucket: List of packages to upload before test runs public: If True, make the bucket publicly accessible + versioned: If True, enable versioning on the bucket before populating """ def decorator(test_func): def wrapper(): bucket = str(uuid.uuid4()) server.succeed(f"mc mb minio/{bucket}") - if public: - server.succeed(f"mc anonymous set download minio/{bucket}") try: + if public: + server.succeed(f"mc anonymous set download minio/{bucket}") + if versioned: + server.succeed(f"mc version enable minio/{bucket}") if populate_bucket: store_url = make_s3_url(bucket) for pkg in populate_bucket: @@ -597,6 +598,47 @@ in print(" ✓ File content verified correct (hash matches)") + @setup_s3(populate_bucket=[PKGS['A']], versioned=True) + def test_versioned_urls(bucket): + """Test that versionId parameter is accepted in S3 URLs""" + print("\n=== Testing Versioned URLs ===") + + # Get the nix-cache-info file + cache_info_url = make_s3_url(bucket, path="/nix-cache-info") + + # Fetch without versionId should work + client.succeed( + f"{ENV_WITH_CREDS} nix eval --impure --expr " + f"'builtins.fetchurl {{ name = \"cache-info\"; url = \"{cache_info_url}\"; }}'" + ) + print(" ✓ Fetch without versionId works") + + # List versions to get a version ID + # MinIO output format: [timestamp] size tier versionId versionNumber method filename + versions_output = server.succeed(f"mc ls --versions minio/{bucket}/nix-cache-info") + + # Extract version ID from output (4th field after STANDARD) + import re + version_match = re.search(r'STANDARD\s+(\S+)\s+v\d+', versions_output) + if not version_match: + print(f"Debug: versions output: {versions_output}") + raise Exception("Could not extract version ID from MinIO output") + + version_id = version_match.group(1) + print(f" ✓ Found version ID: {version_id}") + + # Version ID should not be "null" since versioning was enabled before upload + if version_id == "null": + raise Exception("Version ID is 'null' - versioning may not be working correctly") + + # Fetch with versionId parameter + versioned_url = f"{cache_info_url}&versionId={version_id}" + client.succeed( + f"{ENV_WITH_CREDS} nix eval --impure --expr " + f"'builtins.fetchurl {{ name = \"cache-info-versioned\"; url = \"{versioned_url}\"; }}'" + ) + print(" ✓ Fetch with versionId parameter works") + # ============================================================================ # Main Test Execution # ============================================================================ @@ -626,6 +668,7 @@ in test_compression_mixed() test_compression_disabled() test_nix_prefetch_url() + test_versioned_urls() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 0f0d9255c62054b606cb56594526d03b431fed51 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 24 Oct 2025 18:00:05 -0400 Subject: [PATCH 025/155] Clean up JSON utils in a few ways In particular - Remove `get`, it is redundant with `valueAt` and the `get` in `util.hh`. - Remove `nullableValueAt`. It is morally just the function composition `getNullable . valueAt`, not an orthogonal combinator like the others. - `optionalValueAt` return a pointer, not `std::optional`. This also expresses optionality, but without creating a needless copy. This brings it in line with the other combinators which also return references. - Delete `valueAt` and `optionalValueAt` taking the map by value, as we did for `get` in 408c09a1207e1f6bb7367322ceb25d187334673f, which prevents bugs / unnecessary copies. `adl_serializer::from_json` was the one use of `getNullable`. I give it a little static function for the ultimate creation of a `std::optional` it does need to do (after switching it to using `getNullable . valueAt`. That could go in `json-utils.hh` eventually, but I didn't bother for now since only one things needs it. Co-authored-by: Sergei Zimmerman --- src/libfetchers/fetchers.cc | 5 +- src/libstore/derivation-options.cc | 49 ++++++++++++----- src/libstore/nar-info.cc | 10 ++-- src/libutil-tests/json-utils.cc | 62 +++++++++++++--------- src/libutil/include/nix/util/json-utils.hh | 20 ++++--- src/libutil/json-utils.cc | 45 +++------------- 6 files changed, 99 insertions(+), 92 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 324e8884c..c9c0fffa2 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -519,10 +519,11 @@ using namespace nix; fetchers::PublicKey adl_serializer::from_json(const json & json) { fetchers::PublicKey res = {}; - if (auto type = optionalValueAt(json, "type")) + auto & obj = getObject(json); + if (auto * type = optionalValueAt(obj, "type")) res.type = getString(*type); - res.key = getString(valueAt(json, "key")); + res.key = getString(valueAt(obj, "key")); return res; } diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 698485c0d..6afaf0348 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -116,27 +116,29 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt DerivationOptions defaults = {}; if (shouldWarn && parsed) { - if (get(parsed->structuredAttrs, "allowedReferences")) { + auto & structuredAttrs = getObject(parsed->structuredAttrs); + + if (get(structuredAttrs, "allowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); } - if (get(parsed->structuredAttrs, "allowedRequisites")) { + if (get(structuredAttrs, "allowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); } - if (get(parsed->structuredAttrs, "disallowedRequisites")) { + if (get(structuredAttrs, "disallowedRequisites")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); } - if (get(parsed->structuredAttrs, "disallowedReferences")) { + if (get(structuredAttrs, "disallowedReferences")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); } - if (get(parsed->structuredAttrs, "maxSize")) { + if (get(structuredAttrs, "maxSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); } - if (get(parsed->structuredAttrs, "maxClosureSize")) { + if (get(structuredAttrs, "maxClosureSize")) { warn( "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); } @@ -145,11 +147,15 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt return { .outputChecks = [&]() -> OutputChecksVariant { if (parsed) { + auto & structuredAttrs = getObject(parsed->structuredAttrs); + std::map res; - if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) { - for (auto & [outputName, output] : getObject(*outputChecks)) { + if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { + for (auto & [outputName, output_] : getObject(*outputChecks)) { OutputChecks checks; + auto & output = getObject(output_); + if (auto maxSize = get(output, "maxSize")) checks.maxSize = maxSize->get(); @@ -195,7 +201,9 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt std::map res; if (parsed) { - if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { + auto & structuredAttrs = getObject(parsed->structuredAttrs); + + if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) { for (auto & [outputName, output] : getObject(*udr)) { if (!output.is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName); @@ -226,7 +234,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt std::map ret; if (parsed) { - auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); + auto e = optionalValueAt(getObject(parsed->structuredAttrs), "exportReferencesGraph"); if (!e || !e->is_object()) return ret; for (auto & [key, value] : getObject(*e)) { @@ -333,8 +341,10 @@ namespace nlohmann { using namespace nix; -DerivationOptions adl_serializer::from_json(const json & json) +DerivationOptions adl_serializer::from_json(const json & json_) { + auto & json = getObject(json_); + return { .outputChecks = [&]() -> OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); @@ -397,13 +407,24 @@ void adl_serializer::to_json(json & json, const DerivationOpt json["allowSubstitutes"] = o.allowSubstitutes; } -DerivationOptions::OutputChecks adl_serializer::from_json(const json & json) +template +static inline std::optional ptrToOwned(const json * ptr) { + if (ptr) + return std::optional{*ptr}; + else + return std::nullopt; +} + +DerivationOptions::OutputChecks adl_serializer::from_json(const json & json_) +{ + auto & json = getObject(json_); + return { .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), - .allowedReferences = nullableValueAt(json, "allowedReferences"), + .allowedReferences = ptrToOwned(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")), - .allowedRequisites = nullableValueAt(json, "allowedRequisites"), + .allowedRequisites = ptrToOwned(getNullable(valueAt(json, "allowedRequisites"))), .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")), }; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1e7c48287..6f1abb273 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -159,17 +159,19 @@ NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path, UnkeyedValidPathInfo::fromJSON(store, json), }}; + auto & obj = getObject(json); + if (json.contains("url")) - res.url = getString(valueAt(json, "url")); + res.url = getString(valueAt(obj, "url")); if (json.contains("compression")) - res.compression = getString(valueAt(json, "compression")); + res.compression = getString(valueAt(obj, "compression")); if (json.contains("downloadHash")) - res.fileHash = Hash::parseAny(getString(valueAt(json, "downloadHash")), std::nullopt); + res.fileHash = Hash::parseAny(getString(valueAt(obj, "downloadHash")), std::nullopt); if (json.contains("downloadSize")) - res.fileSize = getUnsigned(valueAt(json, "downloadSize")); + res.fileSize = getUnsigned(valueAt(obj, "downloadSize")); return res; } diff --git a/src/libutil-tests/json-utils.cc b/src/libutil-tests/json-utils.cc index 7d02894c6..b5c011355 100644 --- a/src/libutil-tests/json-utils.cc +++ b/src/libutil-tests/json-utils.cc @@ -70,7 +70,7 @@ TEST(valueAt, simpleObject) auto nested = R"({ "hello": { "world": "" } })"_json; - ASSERT_EQ(valueAt(valueAt(getObject(nested), "hello"), "world"), ""); + ASSERT_EQ(valueAt(getObject(valueAt(getObject(nested), "hello")), "world"), ""); } TEST(valueAt, missingKey) @@ -119,10 +119,12 @@ TEST(getArray, wrongAssertions) { auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; - ASSERT_THROW(getArray(valueAt(json, "object")), Error); - ASSERT_THROW(getArray(valueAt(json, "string")), Error); - ASSERT_THROW(getArray(valueAt(json, "int")), Error); - ASSERT_THROW(getArray(valueAt(json, "boolean")), Error); + auto & obj = getObject(json); + + ASSERT_THROW(getArray(valueAt(obj, "object")), Error); + ASSERT_THROW(getArray(valueAt(obj, "string")), Error); + ASSERT_THROW(getArray(valueAt(obj, "int")), Error); + ASSERT_THROW(getArray(valueAt(obj, "boolean")), Error); } TEST(getString, rightAssertions) @@ -136,10 +138,12 @@ TEST(getString, wrongAssertions) { auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; - ASSERT_THROW(getString(valueAt(json, "object")), Error); - ASSERT_THROW(getString(valueAt(json, "array")), Error); - ASSERT_THROW(getString(valueAt(json, "int")), Error); - ASSERT_THROW(getString(valueAt(json, "boolean")), Error); + auto & obj = getObject(json); + + ASSERT_THROW(getString(valueAt(obj, "object")), Error); + ASSERT_THROW(getString(valueAt(obj, "array")), Error); + ASSERT_THROW(getString(valueAt(obj, "int")), Error); + ASSERT_THROW(getString(valueAt(obj, "boolean")), Error); } TEST(getIntegralNumber, rightAssertions) @@ -156,18 +160,20 @@ TEST(getIntegralNumber, wrongAssertions) auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "signed": -256, "large": 128, "boolean": false })"_json; - ASSERT_THROW(getUnsigned(valueAt(json, "object")), Error); - ASSERT_THROW(getUnsigned(valueAt(json, "array")), Error); - ASSERT_THROW(getUnsigned(valueAt(json, "string")), Error); - ASSERT_THROW(getUnsigned(valueAt(json, "boolean")), Error); - ASSERT_THROW(getUnsigned(valueAt(json, "signed")), Error); + auto & obj = getObject(json); - ASSERT_THROW(getInteger(valueAt(json, "object")), Error); - ASSERT_THROW(getInteger(valueAt(json, "array")), Error); - ASSERT_THROW(getInteger(valueAt(json, "string")), Error); - ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); - ASSERT_THROW(getInteger(valueAt(json, "large")), Error); - ASSERT_THROW(getInteger(valueAt(json, "signed")), Error); + ASSERT_THROW(getUnsigned(valueAt(obj, "object")), Error); + ASSERT_THROW(getUnsigned(valueAt(obj, "array")), Error); + ASSERT_THROW(getUnsigned(valueAt(obj, "string")), Error); + ASSERT_THROW(getUnsigned(valueAt(obj, "boolean")), Error); + ASSERT_THROW(getUnsigned(valueAt(obj, "signed")), Error); + + ASSERT_THROW(getInteger(valueAt(obj, "object")), Error); + ASSERT_THROW(getInteger(valueAt(obj, "array")), Error); + ASSERT_THROW(getInteger(valueAt(obj, "string")), Error); + ASSERT_THROW(getInteger(valueAt(obj, "boolean")), Error); + ASSERT_THROW(getInteger(valueAt(obj, "large")), Error); + ASSERT_THROW(getInteger(valueAt(obj, "signed")), Error); } TEST(getBoolean, rightAssertions) @@ -181,24 +187,28 @@ TEST(getBoolean, wrongAssertions) { auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; - ASSERT_THROW(getBoolean(valueAt(json, "object")), Error); - ASSERT_THROW(getBoolean(valueAt(json, "array")), Error); - ASSERT_THROW(getBoolean(valueAt(json, "string")), Error); - ASSERT_THROW(getBoolean(valueAt(json, "int")), Error); + auto & obj = getObject(json); + + ASSERT_THROW(getBoolean(valueAt(obj, "object")), Error); + ASSERT_THROW(getBoolean(valueAt(obj, "array")), Error); + ASSERT_THROW(getBoolean(valueAt(obj, "string")), Error); + ASSERT_THROW(getBoolean(valueAt(obj, "int")), Error); } TEST(optionalValueAt, existing) { auto json = R"({ "string": "ssh-rsa" })"_json; - ASSERT_EQ(optionalValueAt(json, "string"), std::optional{"ssh-rsa"}); + auto * ptr = optionalValueAt(getObject(json), "string"); + ASSERT_TRUE(ptr); + ASSERT_EQ(*ptr, R"("ssh-rsa")"_json); } TEST(optionalValueAt, empty) { auto json = R"({})"_json; - ASSERT_EQ(optionalValueAt(json, "string"), std::nullopt); + ASSERT_EQ(optionalValueAt(getObject(json), "string"), nullptr); } TEST(getNullable, null) diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 4b5fb4b21..51ebb2b6c 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include "nix/util/error.hh" #include "nix/util/types.hh" @@ -12,20 +11,25 @@ namespace nix { enum struct ExperimentalFeature; -const nlohmann::json * get(const nlohmann::json & map, const std::string & key); - -nlohmann::json * get(nlohmann::json & map, const std::string & key); - /** * Get the value of a json object at a key safely, failing with a nice * error if the key does not exist. * * Use instead of nlohmann::json::at() to avoid ugly exceptions. */ -const nlohmann::json & valueAt(const nlohmann::json::object_t & map, const std::string & key); +const nlohmann::json & valueAt(const nlohmann::json::object_t & map, std::string_view key); -std::optional optionalValueAt(const nlohmann::json::object_t & value, const std::string & key); -std::optional nullableValueAt(const nlohmann::json::object_t & value, const std::string & key); +/** + * @return A pointer to the value assiocated with `key` if `value` + * contains `key`, otherwise return `nullptr` (not JSON `null`!). + */ +const nlohmann::json * optionalValueAt(const nlohmann::json::object_t & value, std::string_view key); + +/** + * Prevents bugs; see `get` for the same trick. + */ +const nlohmann::json & valueAt(nlohmann::json::object_t && map, std::string_view key) = delete; +const nlohmann::json * optionalValueAt(nlohmann::json::object_t && value, std::string_view key) = delete; /** * Downcast the json object, failing with a nice error if the conversion fails. diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 74b3b27cc..1502384e9 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,52 +1,21 @@ #include "nix/util/json-utils.hh" #include "nix/util/error.hh" #include "nix/util/types.hh" -#include -#include -#include +#include "nix/util/util.hh" namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +const nlohmann::json & valueAt(const nlohmann::json::object_t & map, std::string_view key) { - auto i = map.find(key); - if (i == map.end()) - return nullptr; - return &*i; -} - -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) - return nullptr; - return &*i; -} - -const nlohmann::json & valueAt(const nlohmann::json::object_t & map, const std::string & key) -{ - if (!map.contains(key)) + if (auto * p = optionalValueAt(map, key)) + return *p; + else throw Error("Expected JSON object to contain key '%s' but it doesn't: %s", key, nlohmann::json(map).dump()); - - return map.at(key); } -std::optional optionalValueAt(const nlohmann::json::object_t & map, const std::string & key) +const nlohmann::json * optionalValueAt(const nlohmann::json::object_t & map, std::string_view key) { - if (!map.contains(key)) - return std::nullopt; - - return std::optional{map.at(key)}; -} - -std::optional nullableValueAt(const nlohmann::json::object_t & map, const std::string & key) -{ - auto value = valueAt(map, key); - - if (value.is_null()) - return std::nullopt; - - return std::optional{std::move(value)}; + return get(map, key); } const nlohmann::json * getNullable(const nlohmann::json & value) From 7e53afd8b94bf4a42a2959c7db2a0ce491865d59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 25 Oct 2025 13:22:59 -0400 Subject: [PATCH 026/155] Use types to show that structured attrs are always JSON objects Before we just had partial code accessing it. Now, we use `nlohmann::json::object_t`, which is a `std::map`, to enforce this by construction. --- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-env-desugar.cc | 2 +- src/libstore/derivation-options.cc | 21 ++++++++++--------- .../include/nix/store/parsed-derivations.hh | 6 +++--- src/libstore/parsed-derivations.cc | 9 ++++---- src/nix/nix-build/nix-build.cc | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5f06bf009..94dd5cb19 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1374,7 +1374,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName pos, "while evaluating the `__structuredAttrs` " "attribute passed to builtins.derivationStrict")) - jsonObject = StructuredAttrs{.structuredAttrs = json::object()}; + jsonObject = StructuredAttrs{}; /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; diff --git a/src/libstore/build/derivation-env-desugar.cc b/src/libstore/build/derivation-env-desugar.cc index d6e002d91..8d552fc4d 100644 --- a/src/libstore/build/derivation-env-desugar.cc +++ b/src/libstore/build/derivation-env-desugar.cc @@ -25,7 +25,7 @@ DesugaredEnv DesugaredEnv::create( if (drv.structuredAttrs) { auto json = drv.structuredAttrs->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs); res.atFileEnvPair("NIX_ATTRS_SH_FILE", ".attrs.sh") = StructuredAttrs::writeShell(json); - res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = json.dump(); + res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = static_cast(std::move(json)).dump(); } else { /* In non-structured mode, set all bindings either directory in the environment or via a file, as specified by diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 6afaf0348..bd9704b44 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -22,9 +22,9 @@ getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std:: if (i == parsed->structuredAttrs.end()) return {}; else { - if (!i->is_string()) + if (!i->second.is_string()) throw Error("attribute '%s' of must be a string", name); - return i->get(); + return i->second.get(); } } else { auto i = env.find(name); @@ -42,9 +42,9 @@ static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, c if (i == parsed->structuredAttrs.end()) return def; else { - if (!i->is_boolean()) + if (!i->second.is_boolean()) throw Error("attribute '%s' must be a Boolean", name); - return i->get(); + return i->second.get(); } } else { auto i = env.find(name); @@ -63,10 +63,11 @@ getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std: if (i == parsed->structuredAttrs.end()) return {}; else { - if (!i->is_array()) + if (!i->second.is_array()) throw Error("attribute '%s' must be a list of strings", name); + auto & a = getArray(i->second); Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { + for (auto j = a.begin(); j != a.end(); ++j) { if (!j->is_string()) throw Error("attribute '%s' must be a list of strings", name); res.push_back(j->get()); @@ -116,7 +117,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt DerivationOptions defaults = {}; if (shouldWarn && parsed) { - auto & structuredAttrs = getObject(parsed->structuredAttrs); + auto & structuredAttrs = parsed->structuredAttrs; if (get(structuredAttrs, "allowedReferences")) { warn( @@ -147,7 +148,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt return { .outputChecks = [&]() -> OutputChecksVariant { if (parsed) { - auto & structuredAttrs = getObject(parsed->structuredAttrs); + auto & structuredAttrs = parsed->structuredAttrs; std::map res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { @@ -201,7 +202,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt std::map res; if (parsed) { - auto & structuredAttrs = getObject(parsed->structuredAttrs); + auto & structuredAttrs = parsed->structuredAttrs; if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) { for (auto & [outputName, output] : getObject(*udr)) { @@ -234,7 +235,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt std::map ret; if (parsed) { - auto e = optionalValueAt(getObject(parsed->structuredAttrs), "exportReferencesGraph"); + auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); if (!e || !e->is_object()) return ret; for (auto & [key, value] : getObject(*e)) { diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index edef1b2d2..52e97b0e7 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -18,7 +18,7 @@ struct StructuredAttrs { static constexpr std::string_view envVarName{"__json"}; - nlohmann::json structuredAttrs; + nlohmann::json::object_t structuredAttrs; bool operator==(const StructuredAttrs &) const = default; @@ -45,7 +45,7 @@ struct StructuredAttrs */ static void checkKeyNotInUse(const StringPairs & env); - nlohmann::json prepareStructuredAttrs( + nlohmann::json::object_t prepareStructuredAttrs( Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, @@ -62,7 +62,7 @@ struct StructuredAttrs * `prepareStructuredAttrs`, *not* the original `structuredAttrs` * field. */ - static std::string writeShell(const nlohmann::json & prepared); + static std::string writeShell(const nlohmann::json::object_t & prepared); }; } // namespace nix diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 9e8d44d6e..8d147f65f 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -33,7 +33,8 @@ std::optional StructuredAttrs::tryExtract(StringPairs & env) std::pair StructuredAttrs::unparse() const { - return {envVarName, structuredAttrs.dump()}; + // TODO don't copy the JSON object just to dump it. + return {envVarName, static_cast(structuredAttrs).dump()}; } void StructuredAttrs::checkKeyNotInUse(const StringPairs & env) @@ -97,7 +98,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa return jsonList; } -nlohmann::json StructuredAttrs::prepareStructuredAttrs( +nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs( Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, @@ -120,7 +121,7 @@ nlohmann::json StructuredAttrs::prepareStructuredAttrs( return json; } -std::string StructuredAttrs::writeShell(const nlohmann::json & json) +std::string StructuredAttrs::writeShell(const nlohmann::json::object_t & json) { auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { @@ -144,7 +145,7 @@ std::string StructuredAttrs::writeShell(const nlohmann::json & json) std::string jsonSh; - for (auto & [key, value] : json.items()) { + for (auto & [key, value] : json) { if (!std::regex_match(key, shVarName)) continue; diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index eef97aa19..8aced503b 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -600,7 +600,7 @@ static void main_nix_build(int argc, char ** argv) structuredAttrsRC = StructuredAttrs::writeShell(json); auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); - writeFile(attrsJSON, json.dump()); + writeFile(attrsJSON, static_cast(std::move(json)).dump()); auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); writeFile(attrsSH, structuredAttrsRC); From 9d4d10954a06a9c37bb88fb702e084e4634ebd3c Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Oct 2025 00:05:39 +0100 Subject: [PATCH 027/155] diff-closures: print sizes with dynamic unit --- src/libutil-tests/util.cc | 1 + src/libutil/include/nix/util/util.hh | 2 +- src/libutil/util.cc | 9 +++++---- src/nix/diff-closures.cc | 3 +-- src/nix/path-info.cc | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libutil-tests/util.cc b/src/libutil-tests/util.cc index c48b97e8e..32114d9da 100644 --- a/src/libutil-tests/util.cc +++ b/src/libutil-tests/util.cc @@ -158,6 +158,7 @@ TEST(renderSize, misc) ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(-1024, true), " -1.0 KiB"); ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 26f03938a..cb1c9694d 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -104,7 +104,7 @@ N string2IntWithUnitPrefix(std::string_view s) * GiB`. If `align` is set, the number will be right-justified by * padding with spaces on the left. */ -std::string renderSize(uint64_t value, bool align = false); +std::string renderSize(int64_t value, bool align = false); /** * Parse a string into a float. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 383a904ad..69826070c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -132,15 +132,16 @@ std::optional string2Float(const std::string_view s) template std::optional string2Float(const std::string_view s); template std::optional string2Float(const std::string_view s); -std::string renderSize(uint64_t value, bool align) +std::string renderSize(int64_t value, bool align) { static const std::array prefixes{{'K', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'}}; size_t power = 0; - double res = value; - while (res > 1024 && power < prefixes.size()) { + double abs_value = std::abs(value); + while (abs_value > 1024 && power < prefixes.size()) { ++power; - res /= 1024; + abs_value /= 1024; } + double res = (double) value / std::pow(1024.0, power); return fmt(align ? "%6.1f %ciB" : "%.1f %ciB", power == 0 ? res / 1024 : res, prefixes.at(power)); } diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index cbf842e5c..d36a21d74 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -107,8 +107,7 @@ void printClosureDiff( if (!removed.empty() || !added.empty()) items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) - items.push_back( - fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); + items.push_back(fmt("%s%s" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, renderSize(sizeDelta))); logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); } } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index fef3ae120..146b775e5 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -141,7 +141,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON void printSize(std::ostream & str, uint64_t value) { if (humanReadable) - str << fmt("\t%s", renderSize(value, true)); + str << fmt("\t%s", renderSize((int64_t) value, true)); else str << fmt("\t%11d", value); } From 1f6ac88efc33f896d632dd7fcb6244a43a822097 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 14:10:34 +0100 Subject: [PATCH 028/155] Mark some fields in EvalState as const --- src/libexpr/eval.cc | 12 ++++++++---- src/libexpr/include/nix/expr/eval.hh | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a00f4ddf..a6973f590 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3067,7 +3067,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path) return parseExprFromFile(path, staticBaseEnv); } -Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +Expr * EvalState::parseExprFromFile(const SourcePath & path, const std::shared_ptr & staticEnv) { auto buffer = path.resolveSymlinks().readFile(); // readFile hopefully have left some extra space for terminators @@ -3075,8 +3075,8 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +Expr * EvalState::parseExprFromString( + std::string s_, const SourcePath & basePath, const std::shared_ptr & staticEnv) { // NOTE this method (and parseStdin) must take care to *fully copy* their input // into their respective Pos::Origin until the parser stops overwriting its input @@ -3210,7 +3210,11 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat } Expr * EvalState::parse( - char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr & staticEnv) + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + const std::shared_ptr & staticEnv) { DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath DocCommentMap * docComments = &tmpDocComments; diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 76ce62b87..2640dae39 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -418,7 +418,7 @@ public: RootValue vImportedDrvToDerivation = nullptr; - ref inputCache; + const ref inputCache; /** * Debugger @@ -471,18 +471,18 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - ref> srcToStore; + const ref> srcToStore; /** * A cache that maps paths to "resolved" paths for importing Nix * expressions, i.e. `/foo` to `/foo/default.nix`. */ - ref> importResolutionCache; + const ref> importResolutionCache; /** * A cache from resolved paths to values. */ - ref, @@ -592,12 +592,13 @@ public: * Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const SourcePath & path); - Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv); + Expr * parseExprFromFile(const SourcePath & path, const std::shared_ptr & staticEnv); /** * Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr & staticEnv); + Expr * + parseExprFromString(std::string s, const SourcePath & basePath, const std::shared_ptr & staticEnv); Expr * parseExprFromString(std::string s, const SourcePath & basePath); Expr * parseStdin(); @@ -766,7 +767,7 @@ public: #if NIX_USE_BOEHMGC /** A GC root for the baseEnv reference. */ - std::shared_ptr baseEnvP; + const std::shared_ptr baseEnvP; #endif public: @@ -780,7 +781,7 @@ public: /** * The same, but used during parsing to resolve variables. */ - std::shared_ptr staticBaseEnv; // !!! should be private + const std::shared_ptr staticBaseEnv; // !!! should be private /** * Internal primops not exposed to the user. @@ -862,7 +863,7 @@ private: size_t length, Pos::Origin origin, const SourcePath & basePath, - std::shared_ptr & staticEnv); + const std::shared_ptr & staticEnv); /** * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run From fdc5600fa71eb13c547ed9909d71b44d41488fc0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 14:11:59 +0100 Subject: [PATCH 029/155] makeRegexCache(): Return a ref --- src/libexpr/include/nix/expr/eval.hh | 5 +++-- src/libexpr/primops.cc | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 2640dae39..0c7f9cf09 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -191,7 +191,7 @@ std::ostream & operator<<(std::ostream & os, const ValueType t); struct RegexCache; -std::shared_ptr makeRegexCache(); +ref makeRegexCache(); struct DebugTrace { @@ -372,6 +372,7 @@ public: const fetchers::Settings & fetchSettings; const EvalSettings & settings; + SymbolTable symbols; PosTable positions; @@ -504,7 +505,7 @@ private: /** * Cache used by prim_match(). */ - std::shared_ptr regexCache; + const ref regexCache; public: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5f06bf009..d87825a6d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4611,9 +4611,9 @@ struct RegexCache } }; -std::shared_ptr makeRegexCache() +ref makeRegexCache() { - return std::make_shared(); + return make_ref(); } void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v) From ec2fd2dc23671463fa06222e60a383123380295b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 27 Oct 2025 16:14:19 +0300 Subject: [PATCH 030/155] libexpr: Speed up BindingsBuilder::finishSizeIfNecessary Instead of iterating over the newly built bindings we can do a cheaper set_intersection to count duplicates or fall back to a per-element binary search over the "base" bindings. This speeds up `hello` evaluation by around 10ms (0.196s -> 0.187s) and `nixos.closures.ec2.x86_64-linux` by 140ms (2.744s -> 2.609s). This addresses a somewhat steep performance regression from 82315c3807f90be8f4728b32c343e6a2f31969e3 that reduced memory requirements of attribute set merges. With this patch we get back around to 2.31 level of eval performance while keeping the memory usage optimization. Also document the optimization a bit more. --- src/libexpr/include/nix/expr/attr-set.hh | 45 +++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/libexpr/include/nix/expr/attr-set.hh b/src/libexpr/include/nix/expr/attr-set.hh index 46eecd9bd..f57302c42 100644 --- a/src/libexpr/include/nix/expr/attr-set.hh +++ b/src/libexpr/include/nix/expr/attr-set.hh @@ -5,6 +5,7 @@ #include "nix/expr/symbol-table.hh" #include +#include #include #include @@ -463,12 +464,48 @@ private: return bindings->baseLayer; } + /** + * If the bindings gets "layered" on top of another we need to recalculate + * the number of unique attributes in the chain. + * + * This is done by either iterating over the base "layer" and the newly added + * attributes and counting duplicates. If the base "layer" is big this approach + * is inefficient and we fall back to doing per-element binary search in the base + * "layer". + */ void finishSizeIfNecessary() { - if (hasBaseLayer()) - /* NOTE: Do not use std::ranges::distance, since Bindings is a sized - range, but we are calculating this size here. */ - bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end()); + if (!hasBaseLayer()) + return; + + auto & base = *bindings->baseLayer; + auto attrs = std::span(bindings->attrs, bindings->numAttrs); + + Bindings::size_type duplicates = 0; + + /* If the base bindings is smaller than the newly added attributes + iterate using std::set_intersection to run in O(|base| + |attrs|) = + O(|attrs|). Otherwise use an O(|attrs| * log(|base|)) per-attr binary + search to check for duplicates. Note that if we are in this code path then + |attrs| <= bindingsUpdateLayerRhsSizeThreshold, which 16 by default. We are + optimizing for the case when a small attribute set gets "layered" on top of + a much larger one. When attrsets are already small it's fine to do a linear + scan, but we should avoid expensive iterations over large "base" attrsets. */ + if (attrs.size() > base.size()) { + std::set_intersection( + base.begin(), + base.end(), + attrs.begin(), + attrs.end(), + boost::make_function_output_iterator([&]([[maybe_unused]] auto && _) { ++duplicates; })); + } else { + for (const auto & attr : attrs) { + if (base.get(attr.name)) + ++duplicates; + } + } + + bindings->numAttrsInChain = base.numAttrsInChain + attrs.size() - duplicates; } public: From 3994e5627fed704fc0a0cd9717f5d6b6f0f455e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jun 2025 12:06:21 +0200 Subject: [PATCH 031/155] nix store copy-sigs: Add docs --- src/nix/sigs.cc | 7 +++++++ src/nix/store-copy-sigs.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/nix/store-copy-sigs.md diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 142421e9c..bb2cd4e6f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -28,6 +28,13 @@ struct CmdCopySigs : StorePathsCommand return "copy store path signatures from substituters"; } + std::string doc() override + { + return +#include "store-copy-sigs.md" + ; + } + void run(ref store, StorePaths && storePaths) override { if (substituterUris.empty()) diff --git a/src/nix/store-copy-sigs.md b/src/nix/store-copy-sigs.md new file mode 100644 index 000000000..678756221 --- /dev/null +++ b/src/nix/store-copy-sigs.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* To copy signatures from a binary cache to the local store: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --recursive /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + ``` + +* To copy signatures from one binary cache to another: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --store file:///tmp/binary-cache \ + --recursive -v \ + /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + imported 2 signatures + ``` + +# Description + +`nix store copy-sigs` copies store path signatures from one store to another. + +It is not advised to copy signatures to binary cache stores. Binary cache signatures are stored in `.narinfo` files. Since these are cached aggressively, clients may not see the new signatures quickly. It is therefore better to set any required signatures when the paths are first uploaded to the binary cache. + +Store paths are processed in parallel. The amount of parallelism is controlled by the [`http-connections`](@docroot@/command-ref/conf-file.md#conf-http-connections) settings. + +)"" From bbfaaf3a20db5c72367171455443d669c23a2a35 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 14:52:18 +0100 Subject: [PATCH 032/155] showHelp(): Use one callFunction --- src/nix/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index ed889a189..74d22e433 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -256,8 +256,8 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); - state.callFunction(*vRes, *vDump, *vRes, noPos); + Value * args[]{&state.getBuiltin("false"), vDump}; + state.callFunction(*vGenerateManpage, args, *vRes, noPos); auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) From fb26285458d2c63e407b4ca78c6d96809963d237 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 14:53:46 +0100 Subject: [PATCH 033/155] Fix #include --- src/nix/env.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/env.cc b/src/nix/env.cc index 0a211399a..a80bcda67 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,6 +1,7 @@ -#include #include +#include + #include "nix/cmd/command.hh" #include "nix/expr/eval.hh" #include "run.hh" From 8c8b706f6b468153d7b8f8b2f0c88bfa75a1d363 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:01:46 +0100 Subject: [PATCH 034/155] Fix an update to a finished value --- src/libexpr-tests/value/print.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index 1959fddf2..6cadbc70a 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -10,7 +10,7 @@ using namespace testing; struct ValuePrintingTests : LibExprTest { template - void test(Value v, std::string_view expected, A... args) + void test(Value & v, std::string_view expected, A... args) { std::stringstream out; v.print(state, out, args...); @@ -625,10 +625,11 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided) vThree.mkInt(3); builder.insert(state.symbols.create("three"), &vThree); - vAttrs.mkAttrs(builder.finish()); + Value vAttrs2; + vAttrs2.mkAttrs(builder.finish()); test( - vAttrs, + vAttrs2, "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }", PrintOptions{.ansiColors = true, .maxAttrs = 1}); } From a91115bf220ad6e8cbac7d977e48c6983e2e91bb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:04:13 +0100 Subject: [PATCH 035/155] Remove unnecessary virtual --- src/libstore/dummy-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 509b7a0b1..6c8cb3480 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -148,7 +148,7 @@ struct DummyStoreImpl : DummyStore /** * The dummy store is incapable of *not* trusting! :) */ - virtual std::optional isTrustedClient() override + std::optional isTrustedClient() override { return Trusted; } From 3742ae061ef797e42596e1e2fc3b066aaf465373 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:04:56 +0100 Subject: [PATCH 036/155] Typo --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9fd7a967b..6b9c6602b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -622,7 +622,7 @@ struct curlFileTransfer : public FileTransfer void quit() { quitting = true; - /* We wil not be processing any more incomming requests */ + /* We wil not be processing any more incoming requests */ while (!incoming.empty()) incoming.pop(); } From 9321669353849228f5beeb8a337053cacaad00d3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:07:01 +0100 Subject: [PATCH 037/155] Make getDefaultCores() static --- src/libstore/globals.cc | 2 +- src/libstore/include/nix/store/globals.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4fdb820a9..8c542b686 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -150,7 +150,7 @@ std::vector getUserConfigFiles() return files; } -unsigned int Settings::getDefaultCores() const +unsigned int Settings::getDefaultCores() { const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); const unsigned int maxCPU = getMaxCPU(); diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 14647c05f..f9be1d482 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -77,7 +77,7 @@ public: Settings(); - unsigned int getDefaultCores() const; + static unsigned int getDefaultCores(); Path nixPrefix; From 17777e3b703af1c119b71a18a9e425f22e9accfe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:07:56 +0100 Subject: [PATCH 038/155] Settings typos --- src/libstore/include/nix/store/globals.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index f9be1d482..0d6f380a2 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -427,7 +427,7 @@ public: R"( If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. - It means that remote build hosts fetches as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; @@ -503,7 +503,7 @@ public: by the Nix account, its group should be the group specified here, and its mode should be `1775`. - If the build users group is empty, builds areperformed under + If the build users group is empty, builds are performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if `NIX_REMOTE` is `daemon`). Obviously, this should not be used @@ -847,8 +847,8 @@ public: 4. The path to the build's scratch directory. This directory exists only if the build was run with `--keep-failed`. - The stderr and stdout output from the diff hook isn't - displayed to the user. Instead, it print to the nix-daemon's log. + The stderr and stdout output from the diff hook isn't displayed + to the user. Instead, it prints to the nix-daemon's log. When using the Nix daemon, `diff-hook` must be set in the `nix.conf` configuration file, and cannot be passed at the command line. From 1af5a98955fafea6bd32f4d53c30e5ebc2d85d5e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:09:03 +0100 Subject: [PATCH 039/155] Document removed WorkerProto ops --- src/libstore/include/nix/store/worker-protocol.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 29d4828c2..aec3820d2 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -152,6 +152,7 @@ enum struct WorkerProto::Op : uint64_t { AddIndirectRoot = 12, SyncWithGC = 13, FindRoots = 14, + // ExportPath = 16, // removed QueryDeriver = 18, // obsolete SetOptions = 19, CollectGarbage = 20, @@ -161,6 +162,7 @@ enum struct WorkerProto::Op : uint64_t { QueryFailedPaths = 24, ClearFailedPaths = 25, QueryPathInfo = 26, + // ImportPaths = 27, // removed QueryDerivationOutputNames = 28, // obsolete QueryPathFromHashPart = 29, QuerySubstitutablePathInfos = 30, From 91cd42511e5bd591991485a3bb7c942acfc37476 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Oct 2025 15:11:16 +0100 Subject: [PATCH 040/155] Introduce MINIMUM_PROTOCOL_VERSION constant --- src/libstore/daemon.cc | 2 +- src/libstore/include/nix/store/worker-protocol.hh | 1 + src/libstore/remote-store.cc | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6d2a5781..e6efd6c09 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1031,7 +1031,7 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (protoVersion < 256 + 18) + if (protoVersion < MINIMUM_PROTOCOL_VERSION) throw Error("the Nix client version is too old"); WorkerProto::BasicServerConnection conn; diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index aec3820d2..6ae5fdcbc 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -13,6 +13,7 @@ namespace nix { /* Note: you generally shouldn't change the protocol version. Define a new `WorkerProto::Feature` instead. */ #define PROTOCOL_VERSION (1 << 8 | 38) +#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0d83aed4c..949a51f18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,7 +73,7 @@ void RemoteStore::initConnection(Connection & conn) try { auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (protoVersion < 256 + 18) + if (protoVersion < MINIMUM_PROTOCOL_VERSION) throw Error("the Nix daemon version is too old"); conn.protoVersion = protoVersion; conn.features = features; From f234633e2707db752a25f5319d5b63389dd162bd Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Oct 2025 00:55:40 +0100 Subject: [PATCH 041/155] refactor(libutil): remove `showBytes()` in favor of `renderSize()` The `showBytes()` function was redundant with `renderSize()` as the latter automatically selects the appropriate unit (KiB, MiB, GiB, etc.) based on the value, whereas `showBytes()` always formatted as MiB regardless of size. Co-authored-by: Bernardo Meurer Costa --- src/libmain/include/nix/main/shared.hh | 2 -- src/libmain/shared.cc | 2 +- src/libstore/optimise-store.cc | 2 +- src/libutil/include/nix/util/util.hh | 2 -- src/libutil/util.cc | 5 ----- 5 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index 47d08a050..43069ba82 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -89,8 +89,6 @@ extern volatile ::sig_atomic_t blockInt; /* GC helpers. */ -std::string showBytes(uint64_t bytes); - struct GCResults; struct PrintFreed diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4b36ec98e..3b88ea0c9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -406,7 +406,7 @@ RunPager::~RunPager() PrintFreed::~PrintFreed() { if (show) - std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), showBytes(results.bytesFreed)); + std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed)); } } // namespace nix diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8f2878136..3e02fa812 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -312,7 +312,7 @@ void LocalStore::optimiseStore() optimiseStore(stats); - printInfo("%s freed by hard-linking %d files", showBytes(stats.bytesFreed), stats.filesLinked); + printInfo("%s freed by hard-linking %d files", renderSize(stats.bytesFreed), stats.filesLinked); } void LocalStore::optimisePath(const Path & path, RepairFlag repair) diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index cb1c9694d..1234937b4 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -333,8 +333,6 @@ struct overloaded : Ts... template overloaded(Ts...) -> overloaded; -std::string showBytes(uint64_t bytes); - /** * Provide an addition operator between strings and string_views * inexplicably omitted from the standard library. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 69826070c..f14bc63ac 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -257,9 +257,4 @@ std::pair getLine(std::string_view s) } } -std::string showBytes(uint64_t bytes) -{ - return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); -} - } // namespace nix From 584a8e8a0085e3597f79d1ac9cf9970f575de32a Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Oct 2025 01:21:02 +0100 Subject: [PATCH 042/155] treewide: replace manual MiB calculations with renderSize --- src/libfetchers/git-utils.cc | 5 +++-- src/libmain/shared.cc | 14 ++++++++------ src/libstore/gc.cc | 5 ++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 215418522..65587b43a 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -9,6 +9,7 @@ #include "nix/util/users.hh" #include "nix/util/fs-sink.hh" #include "nix/util/sync.hh" +#include "nix/util/util.hh" #include #include @@ -530,12 +531,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this auto act = (Activity *) payload; act->result( resFetchStatus, - fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB", + fmt("%d/%d objects received, %d/%d deltas indexed, %s", stats->received_objects, stats->total_objects, stats->indexed_deltas, stats->total_deltas, - stats->received_bytes / (1024.0 * 1024.0))); + renderSize(stats->received_bytes))); return getInterrupted() ? -1 : 0; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3b88ea0c9..19733fb3e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,6 +6,7 @@ #include "nix/main/loggers.hh" #include "nix/main/progress-bar.hh" #include "nix/util/signals.hh" +#include "nix/util/util.hh" #include #include @@ -64,18 +65,19 @@ void printMissing(ref store, const MissingPaths & missing, Verbosity lvl) } if (!missing.willSubstitute.empty()) { - const float downloadSizeMiB = missing.downloadSize / (1024.f * 1024.f); - const float narSizeMiB = missing.narSize / (1024.f * 1024.f); if (missing.willSubstitute.size() == 1) { printMsg( - lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, narSizeMiB); + lvl, + "this path will be fetched (%s download, %s unpacked):", + renderSize(missing.downloadSize), + renderSize(missing.narSize)); } else { printMsg( lvl, - "these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + "these %d paths will be fetched (%s download, %s unpacked):", missing.willSubstitute.size(), - downloadSizeMiB, - narSizeMiB); + renderSize(missing.downloadSize), + renderSize(missing.narSize)); } std::vector willSubstituteSorted = {}; std::for_each(missing.willSubstitute.begin(), missing.willSubstitute.end(), [&](const StorePath & p) { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 47f40ab8e..193247aa2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -5,6 +5,7 @@ #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" +#include "nix/util/util.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "store-config-private.hh" @@ -906,9 +907,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) #endif ; - printInfo( - "note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + printInfo("note: currently hard linking saves %s", renderSize(unsharedSize - actualSize - overhead)); } /* While we're at it, vacuum the database. */ From 6b6ceddf72979d8cd29aa79863f212818ff46385 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jun 2025 11:41:02 +0200 Subject: [PATCH 043/155] nix store copy-sigs: Use http-connections setting to control parallelism Previously it used the `ThreadPool` default, i.e. `std::thread::hardware_concurrency()`. But copying signatures is not primarily CPU-bound so it makes more sense to use the `http-connections` setting (since we're typically copying from/to a binary cache). --- src/nix/sigs.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 142421e9c..b5ad0c133 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" +#include "nix/store/filetransfer.hh" #include @@ -38,7 +39,7 @@ struct CmdCopySigs : StorePathsCommand for (auto & s : substituterUris) substituters.push_back(openStore(s)); - ThreadPool pool; + ThreadPool pool{fileTransferSettings.httpConnections}; std::atomic added{0}; From 5dcfa86910fa6777e33265647f51116e648f2f70 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 01:00:10 +0000 Subject: [PATCH 044/155] refactor(libstore): use string_view in HttpBinaryCacheStore::makeRequest --- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/http-binary-cache-store.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 945fe1834..738db132d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -161,7 +161,7 @@ void HttpBinaryCacheStore::upsertFile( } } -FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path) +FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) { /* Otherwise the last path fragment will get discarded. */ auto cacheUriWithTrailingSlash = config->cacheUri; diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index d8ba72390..ecad09975 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -86,7 +86,7 @@ protected: const std::string & mimeType, uint64_t sizeHint) override; - FileTransferRequest makeRequest(const std::string & path); + FileTransferRequest makeRequest(std::string_view path); void getFile(const std::string & path, Sink & sink) override; From ef8dd58d9bb41d5c4dba1765bb4342f4082fba5a Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 27 Oct 2025 18:31:55 +0100 Subject: [PATCH 045/155] parser.y: use std::move() to avoid unnecessary copies With #14314, in some places in the parser we started using C++ objects directly rather than pointers. In those places lines like `$$ = $1` now imply a copy when we don't need one. This commit changes those to `$$ = std::move($1)` to avoid those copies. --- src/libexpr/parser.y | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 93c944dcf..2257e1b45 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -345,8 +345,8 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $$.emplace_back(state->at(@2), new ExprString(state->alloc, $2)); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $$.emplace_back(state->at(@2), $3); } + { $$ = std::move($1); $$.emplace_back(state->at(@2), new ExprString(state->alloc, $2)); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); } | DOLLAR_CURLY expr '}' { $$.emplace_back(state->at(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$.emplace_back(state->at(@1), new ExprString(state->alloc, $1)); @@ -391,8 +391,8 @@ path_start ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $$.emplace_back(state->at(@2), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $$.emplace_back(state->at(@2), $3); } + : ind_string_parts IND_STR { $$ = std::move($1); $$.emplace_back(state->at(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); } | { } ; @@ -440,9 +440,9 @@ binds1 ; attrs - : attrs attr { $$ = $1; $$.emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } + : attrs attr { $$ = std::move($1); $$.emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } | attrs string_attr - { $$ = $1; + { $$ = std::move($1); std::visit(overloaded { [&](std::string_view str) { $$.emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); }, [&](Expr * expr) { @@ -457,20 +457,20 @@ attrs ; attrpath - : attrpath '.' attr { $$ = $1; $$.push_back(AttrName(state->symbols.create($3))); } + : attrpath '.' attr { $$ = std::move($1); $$.push_back(AttrName(state->symbols.create($3))); } | attrpath '.' string_attr - { $$ = $1; + { $$ = std::move($1); std::visit(overloaded { [&](std::string_view str) { $$.push_back(AttrName(state->symbols.create(str))); }, [&](Expr * expr) { $$.push_back(AttrName(expr)); } - }, $3); + }, std::move($3)); } | attr { $$.push_back(AttrName(state->symbols.create($1))); } | string_attr { std::visit(overloaded { [&](std::string_view str) { $$.push_back(AttrName(state->symbols.create(str))); }, [&](Expr * expr) { $$.push_back(AttrName(expr)); } - }, $1); + }, std::move($1)); } ; @@ -480,7 +480,7 @@ attr ; string_attr - : '"' string_parts '"' { $$ = $2; } + : '"' string_parts '"' { $$ = std::move($2); } | DOLLAR_CURLY expr '}' { $$ = $2; } ; From 50e8d17f3c08a3d929c2a663a9711dca7a656635 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 27 Oct 2025 18:46:30 +0100 Subject: [PATCH 046/155] parser.y: use emplace_back() for vector --- src/libexpr/parser.y | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 2257e1b45..cf563e869 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -440,11 +440,11 @@ binds1 ; attrs - : attrs attr { $$ = std::move($1); $$.emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } + : attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); } | attrs string_attr { $$ = std::move($1); std::visit(overloaded { - [&](std::string_view str) { $$.emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); }, + [&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); }, [&](Expr * expr) { throw ParseError({ .msg = HintFmt("dynamic attributes not allowed in inherit"), @@ -457,19 +457,19 @@ attrs ; attrpath - : attrpath '.' attr { $$ = std::move($1); $$.push_back(AttrName(state->symbols.create($3))); } + : attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); } | attrpath '.' string_attr { $$ = std::move($1); std::visit(overloaded { - [&](std::string_view str) { $$.push_back(AttrName(state->symbols.create(str))); }, - [&](Expr * expr) { $$.push_back(AttrName(expr)); } + [&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); }, + [&](Expr * expr) { $$.emplace_back(expr); } }, std::move($3)); } - | attr { $$.push_back(AttrName(state->symbols.create($1))); } + | attr { $$.emplace_back(state->symbols.create($1)); } | string_attr { std::visit(overloaded { - [&](std::string_view str) { $$.push_back(AttrName(state->symbols.create(str))); }, - [&](Expr * expr) { $$.push_back(AttrName(expr)); } + [&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); }, + [&](Expr * expr) { $$.emplace_back(expr); } }, std::move($1)); } ; From 9e9dfe36df18ddcbae8172bbd31438d3758dece6 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 27 Oct 2025 19:30:17 +0100 Subject: [PATCH 047/155] libexpr: store ExprList data in Exprs::alloc --- src/libexpr/include/nix/expr/nixexpr.hh | 10 ++++++++-- src/libexpr/parser.y | 10 +++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 86ad01504..26d5addd5 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -442,8 +442,14 @@ struct ExprAttrs : Expr struct ExprList : Expr { - std::vector elems; - ExprList() {}; + std::span elems; + + ExprList(std::pmr::polymorphic_allocator & alloc, std::vector exprs) + : elems({alloc.allocate_object(exprs.size()), exprs.size()}) + { + std::ranges::copy(exprs, elems.begin()); + }; + COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env) override; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 93c944dcf..40d93a15b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -129,7 +129,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app %type expr_pipe_from expr_pipe_into -%type expr_list +%type > list %type binds binds1 %type formals formal_set %type formal @@ -334,7 +334,7 @@ expr_simple { $2->pos = CUR_POS; $$ = $2; } | '{' '}' { $$ = new ExprAttrs(CUR_POS); } - | '[' expr_list ']' { $$ = $2; } + | '[' list ']' { $$ = new ExprList(state->alloc, std::move($2)); } ; string_parts @@ -484,9 +484,9 @@ string_attr | DOLLAR_CURLY expr '}' { $$ = $2; } ; -expr_list - : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); } - | { $$ = new ExprList; } +list + : list expr_select { $$ = std::move($1); $$.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); } + | { } ; formal_set From 91b69e9e70213d7bdb3bef314383a832b5c7aac8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 00:15:24 -0400 Subject: [PATCH 048/155] `nlohmann::json` instance and JSON Schema for `ContentAddress` Co-authored-by: Robert Hensing --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/content-address.md | 21 ++++ .../json/fixup-json-schema-generated-doc.sed | 3 + doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/content-address-v1 | 1 + .../json/schema/content-address-v1.yaml | 55 +++++++++++ .../protocols/json/schema/derivation-v3.yaml | 17 +--- .../json/schema/deriving-path-v1.yaml | 4 +- .../source/protocols/json/schema/hash-v1.yaml | 4 +- src/json-schema-checks/content-address | 1 + src/json-schema-checks/meson.build | 10 +- src/json-schema-checks/package.nix | 1 + src/libstore-tests/content-address.cc | 97 +++++++++++++++---- .../data/content-address/nar.json | 8 ++ .../data/content-address/text.json | 8 ++ src/libstore/content-address.cc | 34 +++++++ .../include/nix/store/content-address.hh | 12 +++ 18 files changed, 242 insertions(+), 37 deletions(-) create mode 100644 doc/manual/source/protocols/json/content-address.md create mode 120000 doc/manual/source/protocols/json/schema/content-address-v1 create mode 100644 doc/manual/source/protocols/json/schema/content-address-v1.yaml create mode 120000 src/json-schema-checks/content-address create mode 100644 src/libstore-tests/data/content-address/nar.json create mode 100644 src/libstore-tests/data/content-address/text.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index eb20f8714..140fa9849 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -35,6 +35,7 @@ mkMesonDerivation (finalAttrs: { ../../.version # For example JSON ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/derived-path # Too many different types of files to filter for now ../../doc/manual diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b4796f652..abd9422cd 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -118,6 +118,7 @@ - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) - [Hash](protocols/json/hash.md) + - [Content Address](protocols/json/content-address.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) diff --git a/doc/manual/source/protocols/json/content-address.md b/doc/manual/source/protocols/json/content-address.md new file mode 100644 index 000000000..2284e30aa --- /dev/null +++ b/doc/manual/source/protocols/json/content-address.md @@ -0,0 +1,21 @@ +{{#include content-address-v1-fixed.md}} + +## Examples + +### [Text](@docroot@/store/store-object/content-address.html#method-text) method + +```json +{{#include schema/content-address-v1/text.json}} +``` + +### [Nix Archive](@docroot@/store/store-object/content-address.html#method-nix-archive) method + +```json +{{#include schema/content-address-v1/nar.json}} +``` + + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 126e666e9..27895d42a 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -12,3 +12,6 @@ s/\\`/`/g # As we have more such relative links, more replacements of this nature # should appear below. s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g +s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g +s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g +s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 191ec6dbe..f79667961 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -10,6 +10,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', + 'content-address-v1', 'derivation-v3', 'deriving-path-v1', ] diff --git a/doc/manual/source/protocols/json/schema/content-address-v1 b/doc/manual/source/protocols/json/schema/content-address-v1 new file mode 120000 index 000000000..35a0dd865 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/content-address-v1.yaml b/doc/manual/source/protocols/json/schema/content-address-v1.yaml new file mode 100644 index 000000000..d0f759201 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/content-address-v1.yaml @@ -0,0 +1,55 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json" +title: Content Address +description: | + This schema describes the JSON representation of Nix's `ContentAddress` type, which conveys information about [content-addressing store objects](@docroot@/store/store-object/content-address.md). + + > **Note** + > + > For current methods of content addressing, this data type is a bit suspicious, because it is neither simply a content address of a file system object (the `method` is richer), nor simply a content address of a store object (the `hash` doesn't account for the references). + > It should thus only be used in contexts where the references are also known / otherwise made tamper-resistant. + + + +type: object +properties: + method: + "$ref": "#/$defs/method" + hash: + title: Content Address + description: | + This would be the content-address itself. + + For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/file-system-object/content-address.md), and not of the store object as a whole. + In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods). + "$ref": "./hash-v1.yaml" +required: +- method +- hash +additionalProperties: false +"$defs": + method: + type: string + enum: [flat, nar, text, git] + title: Content-Addressing Method + description: | + A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 7c92d475d..c950b839f 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json" title: Derivation description: | Experimental JSON representation of a Nix derivation (version 3). @@ -154,19 +154,10 @@ properties: The output path, if known in advance. method: - type: string - title: Content addressing method - enum: [flat, nar, text, git] + "$ref": "./content-address-v1.yaml#/$defs/method" description: | For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. - - Valid method strings are: - - - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) - - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) - - [`text`](@docroot@/store/store-object/content-address.md#method-text) - - [`git`](@docroot@/store/store-object/content-address.md#method-git) - + See the linked original definition for further details. hashAlgo: title: Hash algorithm "$ref": "./hash-v1.yaml#/$defs/algorithm" diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml index 9c0350d3d..7fd74941e 100644 --- a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json" title: Deriving Path description: | This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path). diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index 844959bcd..316fb6d73 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -1,5 +1,5 @@ -"$schema": http://json-schema.org/draft-04/schema# -"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json" title: Hash description: | A cryptographic hash value used throughout Nix for content addressing and integrity verification. diff --git a/src/json-schema-checks/content-address b/src/json-schema-checks/content-address new file mode 120000 index 000000000..194a265a1 --- /dev/null +++ b/src/json-schema-checks/content-address @@ -0,0 +1 @@ +../../src/libstore-tests/data/content-address \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 09da8770b..745fb5ffa 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -30,6 +30,14 @@ schemas = [ 'blake3-base64.json', ], }, + { + 'stem' : 'content-address', + 'schema' : schema_dir / 'content-address-v1.yaml', + 'files' : [ + 'text.json', + 'nar.json', + ], + }, { 'stem' : 'derivation', 'schema' : schema_dir / 'derivation-v3.yaml', @@ -73,8 +81,6 @@ foreach schema : schemas stem + '-schema-valid', jv, args : [ - '--map', - './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', 'http://json-schema.org/draft-04/schema', schema_file, ], diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index cf4e4cb19..6a76c8b28 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -21,6 +21,7 @@ mkMesonDerivation (finalAttrs: { ../../.version ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash + ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ./. diff --git a/src/libstore-tests/content-address.cc b/src/libstore-tests/content-address.cc index 51d591c38..0474fb2e0 100644 --- a/src/libstore-tests/content-address.cc +++ b/src/libstore-tests/content-address.cc @@ -1,6 +1,7 @@ #include #include "nix/store/content-address.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { @@ -8,33 +9,93 @@ namespace nix { * ContentAddressMethod::parse, ContentAddressMethod::render * --------------------------------------------------------------------------*/ -TEST(ContentAddressMethod, testRoundTripPrintParse_1) +static auto methods = ::testing::Values( + std::pair{ContentAddressMethod::Raw::Text, "text"}, + std::pair{ContentAddressMethod::Raw::Flat, "flat"}, + std::pair{ContentAddressMethod::Raw::NixArchive, "nar"}, + std::pair{ContentAddressMethod::Raw::Git, "git"}); + +struct ContentAddressMethodTest : ::testing::Test, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1) { - for (ContentAddressMethod cam : { - ContentAddressMethod::Raw::Text, - ContentAddressMethod::Raw::Flat, - ContentAddressMethod::Raw::NixArchive, - ContentAddressMethod::Raw::Git, - }) { - EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); - } + auto & [cam, _] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } -TEST(ContentAddressMethod, testRoundTripPrintParse_2) +TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2) { - for (const std::string_view camS : { - "text", - "flat", - "nar", - "git", - }) { - EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); - } + auto & [cam, camS] = GetParam(); + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); } +INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods); + TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class ContentAddressTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "content-address"; + +public: + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct ContentAddressJsonTest : ContentAddressTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(ContentAddressJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(ContentAddressJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + ContentAddressJSON, + ContentAddressJsonTest, + ::testing::Values( + std::pair{ + "text", + ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "asdf"), + }, + }, + std::pair{ + "nar", + ContentAddress{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "qwer"), + }, + })); + } // namespace nix diff --git a/src/libstore-tests/data/content-address/nar.json b/src/libstore-tests/data/content-address/nar.json new file mode 100644 index 000000000..21e065cd3 --- /dev/null +++ b/src/libstore-tests/data/content-address/nar.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM=" + }, + "method": "nar" +} diff --git a/src/libstore-tests/data/content-address/text.json b/src/libstore-tests/data/content-address/text.json new file mode 100644 index 000000000..04bc8ac20 --- /dev/null +++ b/src/libstore-tests/data/content-address/text.json @@ -0,0 +1,8 @@ +{ + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts=" + }, + "method": "text" +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 9a57e3aa6..497c2c5b4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,7 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" #include "nix/util/split.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -300,3 +301,36 @@ Hash ContentAddressWithReferences::getHash() const } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +ContentAddressMethod adl_serializer::from_json(const json & json) +{ + return ContentAddressMethod::parse(getString(json)); +} + +void adl_serializer::to_json(json & json, const ContentAddressMethod & m) +{ + json = m.render(); +} + +ContentAddress adl_serializer::from_json(const json & json) +{ + auto obj = getObject(json); + return { + .method = adl_serializer::from_json(valueAt(obj, "method")), + .hash = valueAt(obj, "hash"), + }; +} + +void adl_serializer::to_json(json & json, const ContentAddress & ca) +{ + json = { + {"method", ca.method}, + {"hash", ca.hash}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/content-address.hh b/src/libstore/include/nix/store/content-address.hh index 0a3dc79bd..41ccc69ae 100644 --- a/src/libstore/include/nix/store/content-address.hh +++ b/src/libstore/include/nix/store/content-address.hh @@ -6,6 +6,7 @@ #include "nix/store/path.hh" #include "nix/util/file-content-address.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -308,4 +309,15 @@ struct ContentAddressWithReferences Hash getHash() const; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ContentAddressMethod) +JSON_IMPL(nix::ContentAddress) From 3915b3a111ffe42d1ac9c8162b5506fa7678464f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:10:20 +0000 Subject: [PATCH 049/155] feat(libstore/s3-binary-cache-store): implement `abortMultipartUpload()` Implement `abortMultipartUpload()` for cleaning up incomplete multipart uploads on error: - Constructs URL with `?uploadId=ID` query parameter - Issues `DELETE` request to abort the multipart upload --- src/libstore/s3-binary-cache-store.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 5d97fb0fd..98f742c70 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -26,6 +26,14 @@ public: private: ref s3Config; + + /** + * Abort a multipart upload + * + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax + */ + void abortMultipartUpload(std::string_view key, std::string_view uploadId); }; void S3BinaryCacheStore::upsertFile( @@ -37,6 +45,19 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::DELETE; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; From 5e220271e2dbafb5205684354057aeaa4a58a5c6 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 22:38:43 +0000 Subject: [PATCH 050/155] feat(libstore): add scanForReferencesDeep for per-file reference tracking Introduces `scanForReferencesDeep` to provide per-file granularity when scanning for store path references, enabling better diagnostics for cycle detection and `nix why-depends --precise`. --- src/libstore-tests/references.cc | 143 ++++++++++++++++++ .../include/nix/store/path-references.hh | 57 +++++++ src/libstore/path-references.cc | 90 +++++++++++ 3 files changed, 290 insertions(+) diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 27ecad08f..9cecd573e 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -1,4 +1,6 @@ #include "nix/store/references.hh" +#include "nix/store/path-references.hh" +#include "nix/util/memory-source-accessor.hh" #include @@ -79,4 +81,145 @@ TEST(references, scan) } } +TEST(references, scanForReferencesDeep) +{ + using File = MemorySourceAccessor::File; + + // Create store paths to search for + StorePath path1{"dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo"}; + StorePath path2{"zc842j0rz61mjsp3h3wp5ly71ak6qgdn-bar"}; + StorePath path3{"a5cn2i4b83gnsm60d38l3kgb8qfplm11-baz"}; + + StorePathSet refs{path1, path2, path3}; + + std::string_view hash1 = path1.hashPart(); + std::string_view hash2 = path2.hashPart(); + std::string_view hash3 = path3.hashPart(); + + // Create an in-memory file system with various reference patterns + auto accessor = make_ref(); + accessor->root = File::Directory{ + .contents{ + { + // file1.txt: contains hash1 + "file1.txt", + File::Regular{ + .contents = "This file references " + hash1 + " in its content", + }, + }, + { + // file2.txt: contains hash2 and hash3 + "file2.txt", + File::Regular{ + .contents = "Multiple refs: " + hash2 + " and also " + hash3, + }, + }, + { + // file3.txt: contains no references + "file3.txt", + File::Regular{ + .contents = "This file has no store path references at all", + }, + }, + { + // subdir: a subdirectory + "subdir", + File::Directory{ + .contents{ + { + // subdir/file4.txt: contains hash1 again + "file4.txt", + File::Regular{ + .contents = "Subdirectory file with " + hash1, + }, + }, + }, + }, + }, + { + // link1: a symlink that contains a reference in its target + "link1", + File::Symlink{ + .target = hash2 + "-target", + }, + }, + }, + }; + + // Test the callback-based API + { + std::map foundRefs; + + scanForReferencesDeep(*accessor, CanonPath::root, refs, [&](FileRefScanResult result) { + foundRefs[std::move(result.filePath)] = std::move(result.foundRefs); + }); + + // Verify we found the expected references + EXPECT_EQ(foundRefs.size(), 4); // file1, file2, file4, link1 + + // Check file1.txt found path1 + { + CanonPath f1Path("/file1.txt"); + auto it = foundRefs.find(f1Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path1)); + } + + // Check file2.txt found path2 and path3 + { + CanonPath f2Path("/file2.txt"); + auto it = foundRefs.find(f2Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 2); + EXPECT_TRUE(it->second.count(path2)); + EXPECT_TRUE(it->second.count(path3)); + } + + // Check file3.txt is not in results (no refs) + { + CanonPath f3Path("/file3.txt"); + EXPECT_FALSE(foundRefs.count(f3Path)); + } + + // Check subdir/file4.txt found path1 + { + CanonPath f4Path("/subdir/file4.txt"); + auto it = foundRefs.find(f4Path); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path1)); + } + + // Check symlink found path2 + { + CanonPath linkPath("/link1"); + auto it = foundRefs.find(linkPath); + ASSERT_TRUE(it != foundRefs.end()); + EXPECT_EQ(it->second.size(), 1); + EXPECT_TRUE(it->second.count(path2)); + } + } + + // Test the map-based convenience API + { + auto results = scanForReferencesDeep(*accessor, CanonPath::root, refs); + + EXPECT_EQ(results.size(), 4); // file1, file2, file4, link1 + + // Verify all expected files are in the results + EXPECT_TRUE(results.count(CanonPath("/file1.txt"))); + EXPECT_TRUE(results.count(CanonPath("/file2.txt"))); + EXPECT_TRUE(results.count(CanonPath("/subdir/file4.txt"))); + EXPECT_TRUE(results.count(CanonPath("/link1"))); + EXPECT_FALSE(results.count(CanonPath("/file3.txt"))); + + // Verify the references found in each file are correct + EXPECT_EQ(results.at(CanonPath("/file1.txt")), StorePathSet{path1}); + EXPECT_EQ(results.at(CanonPath("/file2.txt")), StorePathSet({path2, path3})); + EXPECT_EQ(results.at(CanonPath("/subdir/file4.txt")), StorePathSet{path1}); + EXPECT_EQ(results.at(CanonPath("/link1")), StorePathSet{path2}); + } +} + } // namespace nix diff --git a/src/libstore/include/nix/store/path-references.hh b/src/libstore/include/nix/store/path-references.hh index 66d0da268..6aa506da4 100644 --- a/src/libstore/include/nix/store/path-references.hh +++ b/src/libstore/include/nix/store/path-references.hh @@ -3,6 +3,10 @@ #include "nix/store/references.hh" #include "nix/store/path.hh" +#include "nix/util/source-accessor.hh" + +#include +#include namespace nix { @@ -21,4 +25,57 @@ public: StorePathSet getResultPaths(); }; +/** + * Result of scanning a single file for references. + */ +struct FileRefScanResult +{ + CanonPath filePath; ///< The file that was scanned + StorePathSet foundRefs; ///< Which store paths were found in this file +}; + +/** + * Scan a store path tree and report which references appear in which files. + * + * This is like scanForReferences() but provides per-file granularity. + * Useful for cycle detection and detailed dependency analysis like `nix why-depends --precise`. + * + * The function walks the tree using the provided accessor and streams each file's + * contents through a RefScanSink to detect hash references. For each file that + * contains at least one reference, a callback is invoked with the file path and + * the set of references found. + * + * Note: This function only searches for the hash part of store paths (e.g., + * "dc04vv14dak1c1r48qa0m23vr9jy8sm0"), not the name part. A store path like + * "/nix/store/dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo" will be detected if the + * hash appears anywhere in the scanned content, regardless of the "-foo" suffix. + * + * @param accessor Source accessor to read the tree + * @param rootPath Root path to scan + * @param refs Set of store paths to search for + * @param callback Called for each file that contains at least one reference + */ +void scanForReferencesDeep( + SourceAccessor & accessor, + const CanonPath & rootPath, + const StorePathSet & refs, + std::function callback); + +/** + * Scan a store path tree and return which references appear in which files. + * + * This is a convenience wrapper around the callback-based scanForReferencesDeep() + * that collects all results into a map for efficient lookups. + * + * Note: This function only searches for the hash part of store paths, not the name part. + * See the callback-based overload for details. + * + * @param accessor Source accessor to read the tree + * @param rootPath Root path to scan + * @param refs Set of store paths to search for + * @return Map from file paths to the set of references found in each file + */ +std::map +scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs); + } // namespace nix diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 8b167e902..3d783bbe4 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,11 +1,15 @@ #include "nix/store/path-references.hh" #include "nix/util/hash.hh" #include "nix/util/archive.hh" +#include "nix/util/source-accessor.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/logging.hh" #include #include #include #include +#include namespace nix { @@ -54,4 +58,90 @@ StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathS return refsSink.getResultPaths(); } +void scanForReferencesDeep( + SourceAccessor & accessor, + const CanonPath & rootPath, + const StorePathSet & refs, + std::function callback) +{ + // Recursive tree walker + auto walk = [&](this auto & self, const CanonPath & path) -> void { + auto stat = accessor.lstat(path); + + switch (stat.type) { + case SourceAccessor::tRegular: { + // Create a fresh sink for each file to independently detect references. + // RefScanSink accumulates found hashes globally - once a hash is found, + // it remains in the result set. If we reused the same sink across files, + // we couldn't distinguish which files contain which references, as a hash + // found in an earlier file wouldn't be reported when found in later files. + PathRefScanSink sink = PathRefScanSink::fromPaths(refs); + + // Scan this file by streaming its contents through the sink + accessor.readFile(path, sink); + + // Get the references found in this file + auto foundRefs = sink.getResultPaths(); + + // Report if we found anything in this file + if (!foundRefs.empty()) { + debug("scanForReferencesDeep: found %d references in %s", foundRefs.size(), path.abs()); + callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)}); + } + break; + } + + case SourceAccessor::tDirectory: { + // Recursively scan directory contents + auto entries = accessor.readDirectory(path); + for (const auto & [name, entryType] : entries) { + self(path / name); + } + break; + } + + case SourceAccessor::tSymlink: { + // Create a fresh sink for the symlink target (same reason as regular files) + PathRefScanSink sink = PathRefScanSink::fromPaths(refs); + + // Scan symlink target for references + auto target = accessor.readLink(path); + sink(std::string_view(target)); + + // Get the references found in this symlink target + auto foundRefs = sink.getResultPaths(); + + if (!foundRefs.empty()) { + debug("scanForReferencesDeep: found %d references in symlink %s", foundRefs.size(), path.abs()); + callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)}); + } + break; + } + + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: + default: + throw Error("file '%s' has an unsupported type", path.abs()); + } + }; + + // Start the recursive walk from the root + walk(rootPath); +} + +std::map +scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs) +{ + std::map results; + + scanForReferencesDeep(accessor, rootPath, refs, [&](FileRefScanResult result) { + results[std::move(result.filePath)] = std::move(result.foundRefs); + }); + + return results; +} + } // namespace nix From 6129aee988132742837d36fd4cf995bfe85b3198 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 22:55:14 +0000 Subject: [PATCH 051/155] refactor(nix/why-depends): use scanForReferencesDeep for --precise mode Replaces manual tree-walking and reference scanning with the new scanForReferencesDeep function. --- src/nix/why-depends.cc | 79 +++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index dc30fabd7..29da9e953 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,5 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" +#include "nix/store/path-references.hh" #include "nix/util/source-accessor.hh" #include "nix/main/shared.hh" @@ -191,7 +192,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions /* Sort the references by distance to `dependency` to ensure that the shortest path is printed first. */ std::multimap refs; - StringSet hashes; + StorePathSet refPaths; for (auto & ref : node.refs) { if (ref == node.path && packagePath != dependencyPath) @@ -200,7 +201,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (node2.dist == inf) continue; refs.emplace(node2.dist, &node2); - hashes.insert(std::string(node2.path.hashPart())); + refPaths.insert(node2.path); } /* For each reference, find the files and symlinks that @@ -209,58 +210,50 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions auto accessor = store->requireStoreObjectAccessor(node.path); - auto visitPath = [&](this auto && recur, const CanonPath & p) -> void { - auto st = accessor->maybeLstat(p); - assert(st); + auto getColour = [&](const std::string & hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; - auto p2 = p.isRoot() ? p.abs() : p.rel(); + if (precise) { + // Use scanForReferencesDeep to find files containing references + scanForReferencesDeep(*accessor, CanonPath::root, refPaths, [&](FileRefScanResult result) { + auto p2 = result.filePath.isRoot() ? result.filePath.abs() : result.filePath.rel(); + auto st = accessor->lstat(result.filePath); - auto getColour = [&](const std::string & hash) { - return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; - }; + if (st.type == SourceAccessor::Type::tRegular) { + auto contents = accessor->readFile(result.filePath); - if (st->type == SourceAccessor::Type::tDirectory) { - auto names = accessor->readDirectory(p); - for (auto & [name, type] : names) - recur(p / name); - } - - else if (st->type == SourceAccessor::Type::tRegular) { - auto contents = accessor->readFile(p); - - for (auto & hash : hashes) { - auto pos = contents.find(hash); - if (pos != std::string::npos) { - size_t margin = 32; - auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back( - fmt("%s: …%s…", + // For each reference found in this file, extract context + for (auto & foundRef : result.foundRefs) { + std::string hash(foundRef.hashPart()); + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt( + "%s: …%s…", p2, hilite( filterPrintable(std::string(contents, pos2, pos - pos2 + hash.size() + margin)), pos - pos2, StorePath::HashLen, getColour(hash)))); + } + } + } else if (st.type == SourceAccessor::Type::tSymlink) { + auto target = accessor->readLink(result.filePath); + + // For each reference found in this symlink, show it + for (auto & foundRef : result.foundRefs) { + std::string hash(foundRef.hashPart()); + auto pos = target.find(hash); + if (pos != std::string::npos) + hits[hash].emplace_back( + fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } - } - - else if (st->type == SourceAccessor::Type::tSymlink) { - auto target = accessor->readLink(p); - - for (auto & hash : hashes) { - auto pos = target.find(hash); - if (pos != std::string::npos) - hits[hash].emplace_back( - fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); - } - } - }; - - // FIXME: should use scanForReferences(). - - if (precise) - visitPath(CanonPath::root); + }); + } for (auto & ref : refs) { std::string hash(ref.second->path.hashPart()); From dd716dc9be9d54df959b951d97c51c9eafa37d4d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Oct 2025 15:48:07 -0400 Subject: [PATCH 052/155] Create default `Store::narFromPath` implementation in terms of `getFSAccessor` This is a good default (the methods that allow for an arbitrary choice of source accessor are generally preferable both to implement and to use). And it also pays its way by allowing us to delete *both* the `DummyStore` and `LocalStore` implementations. --- src/libstore/dummy-store.cc | 12 ------------ src/libstore/include/nix/store/local-fs-store.hh | 1 - src/libstore/include/nix/store/store-api.hh | 2 +- src/libstore/include/nix/store/uds-remote-store.hh | 2 +- src/libstore/local-fs-store.cc | 7 ------- src/libstore/restricted-store.cc | 2 +- src/libstore/ssh-store.cc | 2 +- src/libstore/store-api.cc | 7 +++++++ 8 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 6c8cb3480..1333e0aed 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -258,18 +258,6 @@ struct DummyStoreImpl : DummyStore }); } - void narFromPath(const StorePath & path, Sink & sink) override - { - bool visited = contents.cvisit(path, [&](const auto & kv) { - const auto & [info, accessor] = kv.second; - SourcePath sourcePath(accessor); - dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); - }); - - if (!visited) - throw Error("path '%s' is not valid", printStorePath(path)); - } - void queryRealisationUncached( const DrvOutput & drvOutput, Callback> callback) noexcept override { diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 08f8e1656..100a4110d 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -78,7 +78,6 @@ struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore LocalFSStore(const Config & params); - void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d03e8e010..8fa13de34 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -609,7 +609,7 @@ public: /** * Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink); /** * For each path, if it's a derivation, build it. Building a diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index fe6e486f4..764e8768a 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore void narFromPath(const StorePath & path, Sink & sink) override { - LocalFSStore::narFromPath(path, sink); + Store::narFromPath(path, sink); } /** diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 28069dcaf..1a38cac3b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -112,13 +112,6 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa return std::make_shared(std::move(absPath)); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) -{ - if (!isValidPath(path)) - throw Error("path '%s' is not valid", printStorePath(path)); - dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); -} - const std::string LocalFSStore::drvsLogDir = "drvs"; std::optional LocalFSStore::getBuildLogExact(const StorePath & path) diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 5270f7d10..ef8aaa380 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -226,7 +226,7 @@ void RestrictedStore::narFromPath(const StorePath & path, Sink & sink) { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + Store::narFromPath(path, sink); } void RestrictedStore::ensurePath(const StorePath & path) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a7e28017f..ce973e734 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -143,7 +143,7 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore void narFromPath(const StorePath & path, Sink & sink) override { - return LocalFSStore::narFromPath(path, sink); + return Store::narFromPath(path, sink); } ref getFSAccessor(bool requireValidPath) override diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cdca6a763..08b75c8fa 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -300,6 +300,13 @@ ValidPathInfo Store::addToStoreSlow( return info; } +void Store::narFromPath(const StorePath & path, Sink & sink) +{ + auto accessor = requireStoreObjectAccessor(path); + SourcePath sourcePath{accessor}; + dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); +} + StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); From 234f029940ce9bfa86f6f49604a47561400d9e27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Oct 2025 15:39:58 -0400 Subject: [PATCH 053/155] Add consuming `ref` <-> `std::share_ptr` methods/ctrs This can help churning ref counts when we don't need to. --- src/libutil/include/nix/util/ref.hh | 32 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libutil/include/nix/util/ref.hh b/src/libutil/include/nix/util/ref.hh index 7cf5ef25e..7ba5349a6 100644 --- a/src/libutil/include/nix/util/ref.hh +++ b/src/libutil/include/nix/util/ref.hh @@ -17,6 +17,12 @@ private: std::shared_ptr p; + void assertNonNull() + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + public: using element_type = T; @@ -24,15 +30,19 @@ public: explicit ref(const std::shared_ptr & p) : p(p) { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); + assertNonNull(); + } + + explicit ref(std::shared_ptr && p) + : p(std::move(p)) + { + assertNonNull(); } explicit ref(T * p) : p(p) { - if (!p) - throw std::invalid_argument("null pointer cast to ref"); + assertNonNull(); } T * operator->() const @@ -45,14 +55,22 @@ public: return *p; } - operator std::shared_ptr() const + std::shared_ptr get_ptr() const & { return p; } - std::shared_ptr get_ptr() const + std::shared_ptr get_ptr() && { - return p; + return std::move(p); + } + + /** + * Convenience to avoid explicit `get_ptr()` call in some cases. + */ + operator std::shared_ptr(this auto && self) + { + return std::forward(self).get_ptr(); } template From 28b73cabccc74304d3474aea8c2d06d4c248f811 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 15:04:56 -0400 Subject: [PATCH 054/155] Make reading and writing derivations store methods This allows for different representations. --- src/libstore/derivations.cc | 27 ++++++++++++++++++--- src/libstore/include/nix/store/store-api.hh | 9 +++++-- src/libstore/store-api.cc | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f44bf3e70..20f1d6ca1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -105,7 +105,7 @@ bool BasicDerivation::isBuiltin() const return builder.substr(0, 8) == "builtin:"; } -StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +static auto infoForDerivation(Store & store, const Derivation & drv) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -117,13 +117,32 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repa auto contents = drv.unparse(store, false); auto hash = hashString(HashAlgorithm::SHA256, contents); auto ca = TextInfo{.hash = hash, .references = references}; - auto path = store.makeFixedOutputPathFromCA(suffix, ca); + return std::tuple{ + suffix, + contents, + references, + store.makeFixedOutputPathFromCA(suffix, ca), + }; +} - if (readOnly || settings.readOnlyMode || (store.isValidPath(path) && !repair)) +StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) +{ + if (readOnly || settings.readOnlyMode) { + auto [_x, _y, _z, path] = infoForDerivation(store, drv); + return path; + } else + return store.writeDerivation(drv, repair); +} + +StorePath Store::writeDerivation(const Derivation & drv, RepairFlag repair) +{ + auto [suffix, contents, references, path] = infoForDerivation(*this, drv); + + if (isValidPath(path) && !repair) return path; StringSource s{contents}; - auto path2 = store.addToStoreFromDump( + auto path2 = addToStoreFromDump( s, suffix, FileSerialisationMethod::Flat, diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 8fa13de34..522a9a45f 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -778,15 +778,20 @@ public: */ Derivation derivationFromPath(const StorePath & drvPath); + /** + * Write a derivation to the Nix store, and return its path. + */ + virtual StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair); + /** * Read a derivation (which must already be valid). */ - Derivation readDerivation(const StorePath & drvPath); + virtual Derivation readDerivation(const StorePath & drvPath); /** * Read a derivation from a potentially invalid path. */ - Derivation readInvalidDerivation(const StorePath & drvPath); + virtual Derivation readInvalidDerivation(const StorePath & drvPath); /** * @param [out] out Place in here the set of all store paths in the diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 08b75c8fa..c292e2e43 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1170,7 +1170,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); if (resolvedDrv) - return writeDerivation(*this, *resolvedDrv, NoRepair, true); + return ::nix::writeDerivation(*this, *resolvedDrv, NoRepair, true); } return path; From 136825b4a2700ebbd20f4ba143e9b1819be0537c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 16:00:27 -0400 Subject: [PATCH 055/155] Make Dummy store store derivations separately This makes for more efficiency. Once we have JSON for the dummy store, it will also make for better JSON, too. --- src/libstore-tests/write-derivation.cc | 4 +- src/libstore/dummy-store.cc | 120 ++++++++++++++---- .../include/nix/store/dummy-store-impl.hh | 9 +- 3 files changed, 107 insertions(+), 26 deletions(-) diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index 3f7de05d3..c320f92fa 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -50,8 +50,8 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) EXPECT_EQ(path1, path2); EXPECT_THAT( [&] { writeDerivation(*store, drv, Repair); }, - ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher( - "operation 'addToStoreFromDump' is not supported by store 'dummy://'"))); + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'"))); } } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 1333e0aed..d11fef73f 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -137,12 +137,31 @@ struct DummyStoreImpl : DummyStore void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override { - bool visited = contents.cvisit(path, [&](const auto & kv) { - callback(std::make_shared(StorePath{kv.first}, kv.second.info)); - }); + if (path.isDerivation()) { + if (auto accessor_ = getMemoryFSAccessor(path)) { + ref accessor = ref{std::move(accessor_)}; + /* compute path info on demand */ + auto narHash = + hashPath({accessor, CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + auto info = std::make_shared(path, UnkeyedValidPathInfo{narHash.hash}); + info->narSize = narHash.numBytesDigested; + info->ca = ContentAddress{ + .method = ContentAddressMethod::Raw::Text, + .hash = hashString( + HashAlgorithm::SHA256, + std::get(accessor->root->raw).contents), + }; + callback(std::move(info)); + return; + } + } else { + if (contents.cvisit(path, [&](const auto & kv) { + callback(std::make_shared(StorePath{kv.first}, kv.second.info)); + })) + return; + } - if (!visited) - callback(nullptr); + callback(nullptr); } /** @@ -169,18 +188,25 @@ struct DummyStoreImpl : DummyStore if (checkSigs) throw Error("checking signatures is not supported for '%s' store", config->getHumanReadableURI()); - auto temp = make_ref(); - MemorySink tempSink{*temp}; + auto accessor = make_ref(); + MemorySink tempSink{*accessor}; parseDump(tempSink, source); auto path = info.path; - auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + if (info.path.isDerivation()) { + warn("back compat supporting `addToStore` for inserting derivations in dummy store"); + writeDerivation( + parseDerivation(*this, accessor->readFile(CanonPath::root), Derivation::nameFromPath(info.path))); + return; + } + + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); } @@ -193,6 +219,9 @@ struct DummyStoreImpl : DummyStore const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override { + if (isDerivation(name)) + throw Error("Do not insert derivation into dummy store with `addToStoreFromDump`"); + if (config->readOnly) unsupported("addToStoreFromDump"); @@ -239,17 +268,47 @@ struct DummyStoreImpl : DummyStore auto path = info.path; auto accessor = make_ref(std::move(*temp)); - contents.insert( - {path, - PathInfoAndContents{ - std::move(info), - accessor, - }}); + contents.insert({ + path, + PathInfoAndContents{ + std::move(info), + accessor, + }, + }); wholeStoreView->addObject(path.to_string(), accessor); return path; } + StorePath writeDerivation(const Derivation & drv, RepairFlag repair = NoRepair) override + { + auto drvPath = ::nix::writeDerivation(*this, drv, repair, /*readonly=*/true); + + if (!derivations.contains(drvPath) || repair) { + if (config->readOnly) + unsupported("writeDerivation"); + derivations.insert({drvPath, drv}); + } + + return drvPath; + } + + Derivation readDerivation(const StorePath & drvPath) override + { + if (std::optional res = getConcurrent(derivations, drvPath)) + return *res; + else + throw Error("derivation '%s' is not valid", printStorePath(drvPath)); + } + + /** + * No such thing as an "invalid derivation" with the dummy store + */ + Derivation readInvalidDerivation(const StorePath & drvPath) override + { + return readDerivation(drvPath); + } + void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); @@ -273,13 +332,28 @@ struct DummyStoreImpl : DummyStore callback(nullptr); } - std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override + std::shared_ptr getMemoryFSAccessor(const StorePath & path, bool requireValidPath = true) { - std::shared_ptr res; - contents.cvisit(path, [&](const auto & kv) { res = kv.second.contents.get_ptr(); }); + std::shared_ptr res; + if (path.isDerivation()) + derivations.cvisit(path, [&](const auto & kv) { + /* compute path info on demand */ + auto res2 = make_ref(); + res2->root = MemorySourceAccessor::File::Regular{ + .contents = kv.second.unparse(*this, false), + }; + res = std::move(res2).get_ptr(); + }); + else + contents.cvisit(path, [&](const auto & kv) { res = kv.second.contents.get_ptr(); }); return res; } + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override + { + return getMemoryFSAccessor(path, requireValidPath); + } + ref getFSAccessor(bool requireValidPath) override { return wholeStoreView; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 4c9f54e98..137f81c9b 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/dummy-store.hh" +#include "nix/store/derivations.hh" #include @@ -25,11 +26,17 @@ struct DummyStore : virtual Store }; /** - * This is map conceptually owns the file system objects for each + * This map conceptually owns the file system objects for each * store object. */ boost::concurrent_flat_map contents; + /** + * This map conceptually owns every derivation, allowing us to + * avoid "on-disk drv format" serialization round-trips. + */ + boost::concurrent_flat_map derivations; + /** * The build trace maps the pair of a content-addressing (fixed or * floating) derivations an one of its output to a From 18941a2421f40efad264ad1fc4e07339075491c3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Oct 2025 11:37:14 -0400 Subject: [PATCH 056/155] Optimize `DummyStore::isValidPathUncached` See the API docs for the rationale of why this is needed. --- src/libstore/dummy-store.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index d11fef73f..c45a13cc3 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -164,6 +164,15 @@ struct DummyStoreImpl : DummyStore callback(nullptr); } + /** + * Do this to avoid `queryPathInfoUncached` computing `PathInfo` + * that we don't need just to return a `bool`. + */ + bool isValidPathUncached(const StorePath & path) override + { + return path.isDerivation() ? derivations.contains(path) : Store::isValidPathUncached(path); + } + /** * The dummy store is incapable of *not* trusting! :) */ From ad664ce64e90234e6a0349b7b14f00bc9c82bf8e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Mon, 27 Oct 2025 20:56:54 +0000 Subject: [PATCH 057/155] ci: cancel previous workflow runs on PR updates Add concurrency group configuration to the CI workflow to automatically cancel outdated runs when a PR receives new commits or is force-pushed. This prevents wasting CI resources on superseded code. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a0820903..67e97b188 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,10 @@ on: default: true type: boolean +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: read-all jobs: From 4b6d07d64299e539ba4f421a6589abc4e630c36f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:53:39 +0000 Subject: [PATCH 058/155] feat(libstore/s3-binary-cache-store): implement `createMultipartUpload()` POST to key with `?uploads` query parameter, optionally set `Content-Encoding` header, parse `uploadId` from XML response using regex --- src/libstore/s3-binary-cache-store.cc | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 98f742c70..58cb72776 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -4,6 +4,7 @@ #include #include +#include namespace nix { @@ -27,6 +28,15 @@ public: private: ref s3Config; + /** + * Creates a multipart upload for large objects to S3. + * + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax + */ + std::string createMultipartUpload( + std::string_view key, std::string_view mimeType, std::optional contentEncoding); + /** * Abort a multipart upload * @@ -45,6 +55,39 @@ void S3BinaryCacheStore::upsertFile( HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint); } +std::string S3BinaryCacheStore::createMultipartUpload( + std::string_view key, std::string_view mimeType, std::optional contentEncoding) +{ + auto req = makeRequest(key); + + // setupForS3() converts s3:// to https:// but strips query parameters + // So we call it first, then add our multipart parameters + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploads"] = ""; + req.uri = VerbatimURL(url); + + req.method = HttpMethod::POST; + req.data = ""; + req.mimeType = mimeType; + + if (contentEncoding) { + req.headers.emplace_back("Content-Encoding", *contentEncoding); + } + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + std::regex uploadIdRegex("([^<]+)"); + std::smatch match; + + if (std::regex_search(result.data, match, uploadIdRegex)) { + return match[1]; + } + + throw Error("S3 CreateMultipartUpload response missing "); +} + void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) { auto req = makeRequest(key); From c592090fffde2fc107dec0bfd398ae7a9c0b4f35 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:02:25 +0000 Subject: [PATCH 059/155] feat(libstore/s3-binary-cache-store): implement `uploadPart()` Implement `uploadPart()` for uploading individual parts in S3 multipart uploads: - Constructs URL with `?partNumber=N&uploadId=ID` query parameters - Uploads chunk data with `application/octet-stream` mime type - Extracts and returns `ETag` from response --- src/libstore/s3-binary-cache-store.cc | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 58cb72776..828e75b7c 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -37,6 +37,15 @@ private: std::string createMultipartUpload( std::string_view key, std::string_view mimeType, std::optional contentEncoding); + /** + * Uploads a single part of a multipart upload + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html#API_UploadPart_RequestSyntax + * + * @returns the [ETag](https://en.wikipedia.org/wiki/HTTP_ETag) + */ + std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); + /** * Abort a multipart upload * @@ -88,6 +97,28 @@ std::string S3BinaryCacheStore::createMultipartUpload( throw Error("S3 CreateMultipartUpload response missing "); } +std::string +S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["partNumber"] = std::to_string(partNumber); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.data = std::move(data); + req.mimeType = "application/octet-stream"; + + auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + + if (result.etag.empty()) { + throw Error("S3 UploadPart response missing ETag for part %d", partNumber); + } + + return std::move(result.etag); +} + void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) { auto req = makeRequest(key); From 3775a2a2268bbc18716363e38868e3bf76fd3884 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:22:54 +0000 Subject: [PATCH 060/155] build(deps): bump actions/upload-artifact from 4 to 5 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..18ae4d8bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} - name: Upload coverage reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: coverage-reports path: coverage-reports/ if: ${{ matrix.instrumented }} - name: Upload installer tarball - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: installer-${{matrix.os}} path: out/* From ccc06451df3ca9345977ca4cdf7d412f6603dd90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:35:42 +0000 Subject: [PATCH 061/155] build(deps): bump actions/download-artifact from 5 to 6 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5 to 6. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..10103847a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Download installer tarball - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: installer-${{matrix.os}} path: out From c77317b1a9086b9aa8ff1b22da051e520febe871 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Fri, 24 Oct 2025 23:54:49 +0000 Subject: [PATCH 062/155] feat(libstore/s3-binary-cache-store): implement `completeMultipartUpload()` `completeMultipartUpload()`: Build XML with part numbers and `ETags`, POST to key with `?uploadId` to finalize the multipart upload --- src/libstore/s3-binary-cache-store.cc | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 828e75b7c..178373778 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -5,6 +5,7 @@ #include #include #include +#include namespace nix { @@ -46,6 +47,19 @@ private: */ std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); + struct UploadedPart + { + uint64_t partNumber; + std::string etag; + }; + + /** + * Completes a multipart upload by combining all uploaded parts. + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax + */ + void completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span parts); + /** * Abort a multipart upload * @@ -132,6 +146,34 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ getFileTransfer()->enqueueFileTransfer(req).get(); } +void S3BinaryCacheStore::completeMultipartUpload( + std::string_view key, std::string_view uploadId, std::span parts) +{ + auto req = makeRequest(key); + req.setupForS3(); + + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::POST; + + std::string xml = ""; + for (const auto & part : parts) { + xml += ""; + xml += "" + std::to_string(part.partNumber) + ""; + xml += "" + part.etag + ""; + xml += ""; + } + xml += ""; + + debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + + req.data = xml; + req.mimeType = "text/xml"; + + getFileTransfer()->enqueueFileTransfer(req).get(); +} + StringSet S3BinaryCacheStoreConfig::uriSchemes() { return {"s3"}; From 94965a3a3eeac6574a06a36760e6470977a7c1f9 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 20:15:25 +0000 Subject: [PATCH 063/155] test(nixos): add S3 multipart upload integration tests --- tests/nixos/s3-binary-cache-store.nix | 129 ++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a2ede4572..a07375489 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -34,8 +34,10 @@ in pkgA pkgB pkgC + pkgs.coreutils ]; environment.systemPackages = [ pkgs.minio-client ]; + nix.nixPath = [ "nixpkgs=${pkgs.path}" ]; nix.extraOptions = '' experimental-features = nix-command substituters = @@ -639,6 +641,129 @@ in ) print(" ✓ Fetch with versionId parameter works") + @setup_s3() + def test_multipart_upload_basic(bucket): + """Test basic multipart upload with a large file""" + print("\n--- Test: Multipart Upload Basic ---") + + large_file_size = 10 * 1024 * 1024 + large_pkg = server.succeed( + "nix-store --add $(dd if=/dev/urandom of=/tmp/large-file bs=1M count=10 2>/dev/null && echo /tmp/large-file)" + ).strip() + + chunk_size = 5 * 1024 * 1024 + expected_parts = 3 # 10 MB raw becomes ~10.5 MB compressed (NAR + xz overhead) + + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(5 * 1024 * 1024), + "multipart-chunk-size": str(chunk_size), + } + ) + + print(f" Uploading {large_file_size} byte file (expect {expected_parts} parts)") + output = server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {large_pkg} --debug 2>&1") + + if "using S3 multipart upload" not in output: + raise Exception("Expected multipart upload to be used") + + expected_msg = f"{expected_parts} parts uploaded" + if expected_msg not in output: + print("Debug output:") + print(output) + raise Exception(f"Expected '{expected_msg}' in output") + + print(f" ✓ Multipart upload used with {expected_parts} parts") + + client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' {large_pkg} --no-check-sigs") + verify_packages_in_store(client, large_pkg, should_exist=True) + + print(" ✓ Large file downloaded and verified") + + @setup_s3() + def test_multipart_threshold(bucket): + """Test that files below threshold use regular upload""" + print("\n--- Test: Multipart Threshold Behavior ---") + + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(1024 * 1024 * 1024), + } + ) + + print(" Uploading small file with high threshold") + output = server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']} --debug 2>&1") + + if "using S3 multipart upload" in output: + raise Exception("Should not use multipart for file below threshold") + + if "using S3 regular upload" not in output: + raise Exception("Expected regular upload to be used") + + print(" ✓ Regular upload used for file below threshold") + + client.succeed(f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}") + verify_packages_in_store(client, PKGS['A'], should_exist=True) + + print(" ✓ Small file uploaded and verified") + + @setup_s3() + def test_multipart_with_log_compression(bucket): + """Test multipart upload with compressed build logs""" + print("\n--- Test: Multipart Upload with Log Compression ---") + + # Create a derivation that produces a large text log (12 MB of base64 output) + drv_path = server.succeed( + """ + nix-instantiate --expr ' + let pkgs = import {}; + in derivation { + name = "large-log-builder"; + builder = "/bin/sh"; + args = ["-c" "$coreutils/bin/dd if=/dev/urandom bs=1M count=12 | $coreutils/bin/base64; echo success > $out"]; + coreutils = pkgs.coreutils; + system = builtins.currentSystem; + } + ' + """ + ).strip() + + print(" Building derivation to generate large log") + server.succeed(f"nix-store --realize {drv_path} &>/dev/null") + + # Upload logs with compression and multipart + store_url = make_s3_url( + bucket, + **{ + "multipart-upload": "true", + "multipart-threshold": str(5 * 1024 * 1024), + "multipart-chunk-size": str(5 * 1024 * 1024), + "log-compression": "xz", + } + ) + + print(" Uploading build log with compression and multipart") + output = server.succeed( + f"{ENV_WITH_CREDS} nix store copy-log --to '{store_url}' {drv_path} --debug 2>&1" + ) + + # Should use multipart for the compressed log + if "using S3 multipart upload" not in output or "log/" not in output: + print("Debug output:") + print(output) + raise Exception("Expected multipart upload to be used for compressed log") + + if "parts uploaded" not in output: + print("Debug output:") + print(output) + raise Exception("Expected multipart completion message") + + print(" ✓ Compressed log uploaded with multipart") + # ============================================================================ # Main Test Execution # ============================================================================ @@ -669,6 +794,10 @@ in test_compression_disabled() test_nix_prefetch_url() test_versioned_urls() + # FIXME: enable when multipart fully lands + # test_multipart_upload_basic() + # test_multipart_threshold() + # test_multipart_with_log_compression() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 972915cabd772c4056fc4d08abd0579f1c252147 Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Tue, 28 Oct 2025 09:36:46 +0000 Subject: [PATCH 064/155] docs: remove incorrect claim re gc --print-dead Per #7591, the `nix-store --gc --print-dead` command does not provide any feedback about the amount of disk space that is used by dead store paths. It looks like this has been the case since 7ab68961e (* Garbage collector: added an option `--use-atime' to delete paths in..., 2008-09-17). Update the nix-store documentation to remove the claim that this is function that `nix-store --gc --print-dead` performs. --- doc/manual/source/command-ref/nix-store/gc.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-store/gc.md b/doc/manual/source/command-ref/nix-store/gc.md index f432e00eb..8ec59d906 100644 --- a/doc/manual/source/command-ref/nix-store/gc.md +++ b/doc/manual/source/command-ref/nix-store/gc.md @@ -48,8 +48,7 @@ The behaviour of the collector is also influenced by the configuration file. By default, the collector prints the total number of freed bytes when it -finishes (or when it is interrupted). With `--print-dead`, it prints the -number of bytes that would be freed. +finishes (or when it is interrupted). {{#include ./opt-common.md}} From 5fc0c4f1027f673f76768b2e8659321cedda6834 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 22 Sep 2025 14:07:03 +0200 Subject: [PATCH 065/155] doc: Improve libexpr-c docs - Uses the more explicit `@ingroup` most of the time, to avoid problems with nested groups, and to make group membership more explicit. The division into headers is not great for documentation purposes, so this helps. - More attention for memory management details - Various other improvements to doc comments --- .../source/development/documentation.md | 6 + src/external-api-docs/README.md | 2 +- src/libexpr-c/nix_api_expr.h | 43 ++- src/libexpr-c/nix_api_external.h | 12 +- src/libexpr-c/nix_api_value.h | 250 +++++++++++++----- src/libutil-c/nix_api_util.h | 2 + 6 files changed, 237 insertions(+), 78 deletions(-) diff --git a/doc/manual/source/development/documentation.md b/doc/manual/source/development/documentation.md index a2a54175d..6823780cc 100644 --- a/doc/manual/source/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -240,3 +240,9 @@ $ configurePhase $ ninja src/external-api-docs/html $ xdg-open src/external-api-docs/html/index.html ``` + +If you use direnv, or otherwise want to run `configurePhase` in a transient shell, use: + +```bash +nix-shell -A devShells.x86_64-linux.native-clangStdenv --command 'mesonFlags="$mesonFlags -Ddoc-gen=true"; mesonConfigurePhase' +``` diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md index 8760ac88b..1940cc1c0 100644 --- a/src/external-api-docs/README.md +++ b/src/external-api-docs/README.md @@ -15,7 +15,7 @@ programmatically: 1. Embedding the evaluator 2. Writing language plug-ins -Embedding means you link the Nix C libraries in your program and use them from +Embedding means you link the Nix C API libraries in your program and use them from there. Adding a plug-in means you make a library that gets loaded by the Nix language evaluator, specified through a configuration option. diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 2be739955..3623ee076 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -4,11 +4,14 @@ * @brief Bindings to the Nix language evaluator * * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example. - * @{ */ /** @file * @brief Main entry for the libexpr C bindings */ +/** @defgroup libexpr_init Initialization + * @ingroup libexpr + * @{ + */ #include "nix_api_store.h" #include "nix_api_util.h" @@ -45,7 +48,10 @@ typedef struct nix_eval_state_builder nix_eval_state_builder; */ typedef struct EvalState EvalState; // nix::EvalState +/** @} */ + /** @brief A Nix language value, or thunk that may evaluate to a value. + * @ingroup value * * Values are the primary objects manipulated in the Nix language. * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its @@ -56,7 +62,8 @@ typedef struct EvalState EvalState; // nix::EvalState * * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * - * @see value_manip + * @struct nix_value + * @see value_create, value_extract * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; @@ -65,6 +72,7 @@ NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** * @brief Initialize the Nix language evaluator. + * @ingroup libexpr_init * * This function must be called at least once, * at some point before constructing a EvalState for the first time. @@ -77,6 +85,7 @@ nix_err nix_libexpr_init(nix_c_context * context); /** * @brief Parses and evaluates a Nix expression from a string. + * @ingroup value_create * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -93,6 +102,7 @@ nix_err nix_expr_eval_from_string( /** * @brief Calls a Nix function with an argument. + * @ingroup value_create * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -107,6 +117,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * f /** * @brief Calls a Nix function with multiple arguments. + * @ingroup value_create * * Technically these are functions that return functions. It is common for Nix * functions to be curried, so this function is useful for calling them. @@ -126,10 +137,12 @@ nix_err nix_value_call_multi( /** * @brief Calls a Nix function with multiple arguments. + * @ingroup value_create * * Technically these are functions that return functions. It is common for Nix * functions to be curried, so this function is useful for calling them. * + * @def NIX_VALUE_CALL * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. * @param[out] value The result of the function call. @@ -147,6 +160,7 @@ nix_err nix_value_call_multi( /** * @brief Forces the evaluation of a Nix value. + * @ingroup value_create * * The Nix interpreter is lazy, and not-yet-evaluated values can be * of type NIX_TYPE_THUNK instead of their actual value. @@ -180,18 +194,20 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val /** * @brief Create a new nix_eval_state_builder + * @ingroup libexpr_init * * The settings are initialized to their default value. * Values can be sourced elsewhere with nix_eval_state_builder_load. * * @param[out] context Optional, stores error information * @param[in] store The Nix store to use. - * @return A new nix_eval_state_builder or NULL on failure. + * @return A new nix_eval_state_builder or NULL on failure. Call nix_eval_state_builder_free() when you're done. */ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store); /** * @brief Read settings from the ambient environment + * @ingroup libexpr_init * * Settings are sourced from environment variables and configuration files, * as documented in the Nix manual. @@ -204,6 +220,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil /** * @brief Set the lookup path for `<...>` expressions + * @ingroup libexpr_init * * @param[in] context Optional, stores error information * @param[in] builder The builder to modify. @@ -214,18 +231,21 @@ nix_err nix_eval_state_builder_set_lookup_path( /** * @brief Create a new Nix language evaluator state + * @ingroup libexpr_init * - * Remember to nix_eval_state_builder_free after building the state. + * The builder becomes unusable after this call. Remember to call nix_eval_state_builder_free() + * after building the state. * * @param[out] context Optional, stores error information * @param[in] builder The builder to use and free - * @return A new Nix state or NULL on failure. + * @return A new Nix state or NULL on failure. Call nix_state_free() when you're done. * @see nix_eval_state_builder_new, nix_eval_state_builder_free */ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder); /** * @brief Free a nix_eval_state_builder + * @ingroup libexpr_init * * Does not fail. * @@ -235,19 +255,21 @@ void nix_eval_state_builder_free(nix_eval_state_builder * builder); /** * @brief Create a new Nix language evaluator state + * @ingroup libexpr_init * * For more control, use nix_eval_state_builder * * @param[out] context Optional, stores error information * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. - * @return A new Nix state or NULL on failure. + * @return A new Nix state or NULL on failure. Call nix_state_free() when you're done. * @see nix_state_builder_new */ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); /** * @brief Frees a Nix state. + * @ingroup libexpr_init * * Does not fail. * @@ -256,6 +278,7 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, void nix_state_free(EvalState * state); /** @addtogroup GC + * @ingroup libexpr * @brief Reference counting and garbage collector operations * * The Nix language evaluator uses a garbage collector. To ease C interop, we implement @@ -286,6 +309,9 @@ nix_err nix_gc_incref(nix_c_context * context, const void * object); /** * @brief Decrement the garbage collector reference counter for the given object * + * @deprecated We are phasing out the general nix_gc_decref() in favor of type-specified free functions, such as + * nix_value_decref(). + * * We also provide typed `nix_*_decref` functions, which are * - safer to use * - easier to integrate when deriving bindings @@ -314,12 +340,11 @@ void nix_gc_now(); */ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)); -/** @} */ +/** @} */ // doxygen group GC + // cffi end #ifdef __cplusplus } #endif -/** @} */ - #endif // NIX_API_EXPR_H diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index f4a327281..96c479d57 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -2,11 +2,12 @@ #define NIX_API_EXTERNAL_H /** @ingroup libexpr * @addtogroup Externals - * @brief Deal with external values + * @brief Externals let Nix expressions work with foreign values that aren't part of the normal Nix value data model * @{ */ /** @file * @brief libexpr C bindings dealing with external values + * @see Externals */ #include "nix_api_expr.h" @@ -115,7 +116,7 @@ typedef struct NixCExternalValueDesc * @brief Try to compare two external values * * Optional, the default is always false. - * If the other object was not a Nix C external value, this comparison will + * If the other object was not a Nix C API external value, this comparison will * also return false * @param[in] self the void* passed to nix_create_external_value * @param[in] other the void* passed to the other object's @@ -168,7 +169,7 @@ typedef struct NixCExternalValueDesc /** * @brief Create an external value, that can be given to nix_init_external * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer. + * Call nix_gc_decref() when you're done with the pointer. * * @param[out] context Optional, stores error information * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long @@ -180,10 +181,11 @@ typedef struct NixCExternalValueDesc ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v); /** - * @brief Extract the pointer from a nix c external value. + * @brief Extract the pointer from a Nix C API external value. * @param[out] context Optional, stores error information * @param[in] b The external value - * @returns The pointer, or null if the external value was not from nix c. + * @returns The pointer, valid while the external value is valid, or null if the external value was not from the Nix C + * API. * @see nix_get_external */ void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b); diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 835eaec6e..5bd45da90 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -1,9 +1,6 @@ #ifndef NIX_API_VALUE_H #define NIX_API_VALUE_H -/** @addtogroup libexpr - * @{ - */ /** @file * @brief libexpr C bindings dealing with values */ @@ -20,18 +17,89 @@ extern "C" { #endif // cffi start +/** @defgroup value Value + * @ingroup libexpr + * @brief nix_value type and core operations for working with Nix values + * @see value_create + * @see value_extract + */ + +/** @defgroup value_create Value Creation + * @ingroup libexpr + * @brief Functions for allocating and initializing Nix values + * + * Values are usually created with `nix_alloc_value` followed by `nix_init_*` functions. + * In primop callbacks, allocation is already done and only initialization is needed. + */ + +/** @defgroup value_extract Value Extraction + * @ingroup libexpr + * @brief Functions for extracting data from Nix values + */ + +/** @defgroup primops PrimOps and Builtins + * @ingroup libexpr + */ + // Type definitions +/** @brief Represents the state of a Nix value + * + * Thunk values (NIX_TYPE_THUNK) change to their final, unchanging type when forced. + * + * @see https://nix.dev/manual/nix/latest/language/evaluation.html + * @enum ValueType + * @ingroup value + */ typedef enum { + /** Unevaluated expression + * + * Thunks often contain an expression and closure, but may contain other + * representations too. + * + * Their state is mutable, unlike that of the other types. + */ NIX_TYPE_THUNK, + /** + * A 64 bit signed integer. + */ NIX_TYPE_INT, + /** @brief IEEE 754 double precision floating point number + * @see https://nix.dev/manual/nix/latest/language/types.html#type-float + */ NIX_TYPE_FLOAT, + /** @brief Boolean true or false value + * @see https://nix.dev/manual/nix/latest/language/types.html#type-bool + */ NIX_TYPE_BOOL, + /** @brief String value with context + * + * String content may contain arbitrary bytes, not necessarily UTF-8. + * @see https://nix.dev/manual/nix/latest/language/types.html#type-string + */ NIX_TYPE_STRING, + /** @brief Filesystem path + * @see https://nix.dev/manual/nix/latest/language/types.html#type-path + */ NIX_TYPE_PATH, + /** @brief Null value + * @see https://nix.dev/manual/nix/latest/language/types.html#type-null + */ NIX_TYPE_NULL, + /** @brief Attribute set (key-value mapping) + * @see https://nix.dev/manual/nix/latest/language/types.html#type-attrs + */ NIX_TYPE_ATTRS, + /** @brief Ordered list of values + * @see https://nix.dev/manual/nix/latest/language/types.html#type-list + */ NIX_TYPE_LIST, + /** @brief Function (lambda or builtin) + * @see https://nix.dev/manual/nix/latest/language/types.html#type-function + */ NIX_TYPE_FUNCTION, + /** @brief External value from C++ plugins or C API + * @see Externals + */ NIX_TYPE_EXTERNAL } ValueType; @@ -39,22 +107,41 @@ typedef enum { typedef struct nix_value nix_value; typedef struct EvalState EvalState; +/** @deprecated Use nix_value instead */ [[deprecated("use nix_value instead")]] typedef nix_value Value; // type defs /** @brief Stores an under-construction set of bindings - * @ingroup value_manip + * @ingroup value_create * - * Do not reuse. + * Each builder can only be used once. After calling nix_make_attrs(), the builder + * becomes invalid and must not be used again. Call nix_bindings_builder_free() to release it. + * + * Typical usage pattern: + * 1. Create with nix_make_bindings_builder() + * 2. Insert attributes with nix_bindings_builder_insert() + * 3. Create final attribute set with nix_make_attrs() + * 4. Free builder with nix_bindings_builder_free() + * + * @struct BindingsBuilder * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs * @see nix_bindings_builder_insert */ typedef struct BindingsBuilder BindingsBuilder; /** @brief Stores an under-construction list - * @ingroup value_manip + * @ingroup value_create * - * Do not reuse. + * Each builder can only be used once. After calling nix_make_list(), the builder + * becomes invalid and must not be used again. Call nix_list_builder_free() to release it. + * + * Typical usage pattern: + * 1. Create with nix_make_list_builder() + * 2. Insert elements with nix_list_builder_insert() + * 3. Create final list with nix_make_list() + * 4. Free builder with nix_list_builder_free() + * + * @struct ListBuilder * @see nix_make_list_builder, nix_list_builder_free, nix_make_list * @see nix_list_builder_insert */ @@ -63,25 +150,28 @@ typedef struct ListBuilder ListBuilder; /** @brief PrimOp function * @ingroup primops * - * Owned by the GC - * @see nix_alloc_primop, nix_init_primop + * Can be released with nix_gc_decref() when necessary. + * @struct PrimOp + * @see nix_alloc_primop, nix_init_primop, nix_register_primop */ typedef struct PrimOp PrimOp; /** @brief External Value * @ingroup Externals * - * Owned by the GC + * Can be released with nix_gc_decref() when necessary. + * @struct ExternalValue + * @see nix_create_external_value, nix_init_external, nix_get_external */ typedef struct ExternalValue ExternalValue; /** @brief String without placeholders, and realised store paths + * @struct nix_realised_string + * @see nix_string_realise, nix_realised_string_free */ typedef struct nix_realised_string nix_realised_string; -/** @defgroup primops Adding primops - * @{ - */ /** @brief Function pointer for primops + * @ingroup primops * * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here"). * @@ -97,9 +187,9 @@ typedef void (*PrimOpFun)( void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); /** @brief Allocate a PrimOp + * @ingroup primops * - * Owned by the garbage collector. - * Use nix_gc_decref() when you're done with the returned PrimOp. + * Call nix_gc_decref() when you're done with the returned PrimOp. * * @param[out] context Optional, stores error information * @param[in] fun callback @@ -121,35 +211,38 @@ PrimOp * nix_alloc_primop( void * user_data); /** @brief add a primop to the `builtins` attribute set + * @ingroup primops * * Only applies to States created after this call. * - * Moves your PrimOp content into the global evaluator - * registry, meaning your input PrimOp pointer is no longer usable. - * You are free to remove your references to it, - * after which it will be garbage collected. + * Moves your PrimOp content into the global evaluator registry, meaning + * your input PrimOp pointer becomes invalid. The PrimOp must not be used + * with nix_init_primop() before or after this call, as this would cause + * undefined behavior. + * You must call nix_gc_decref() on the original PrimOp pointer + * after this call to release your reference. * * @param[out] context Optional, stores error information - * @return primop, or null in case of errors - * + * @param[in] primOp PrimOp to register + * @return error code, NIX_OK on success */ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); -/** @} */ // Function prototypes /** @brief Allocate a Nix value + * @ingroup value_create * - * Owned by the GC. Use nix_gc_decref() when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] state nix evaluator state * @return value, or null in case of errors - * */ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); /** * @brief Increment the garbage collector reference counter for the given `nix_value`. + * @ingroup value * * The Nix language evaluator C API keeps track of alive objects by reference counting. * When you're done with a refcounted pointer, call nix_value_decref(). @@ -161,21 +254,19 @@ nix_err nix_value_incref(nix_c_context * context, nix_value * value); /** * @brief Decrement the garbage collector reference counter for the given object + * @ingroup value + * + * When the counter reaches zero, the `nix_value` object becomes invalid. + * The data referenced by `nix_value` may not be deallocated until the memory + * garbage collector has run, but deallocation is not guaranteed. * * @param[out] context Optional, stores error information * @param[in] value The object to stop referencing */ nix_err nix_value_decref(nix_c_context * context, nix_value * value); -/** @addtogroup value_manip Manipulating values - * @brief Functions to inspect and change Nix language values, represented by nix_value. - * @{ - */ -/** @anchor getters - * @name Getters - */ -/**@{*/ /** @brief Get value type + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return type of nix value @@ -183,14 +274,15 @@ nix_err nix_value_decref(nix_c_context * context, nix_value * value); ValueType nix_get_type(nix_c_context * context, const nix_value * value); /** @brief Get type name of value as defined in the evaluator + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return type name, owned string - * @todo way to free the result + * @return type name string, free with free() */ const char * nix_get_typename(nix_c_context * context, const nix_value * value); /** @brief Get boolean value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return true or false, error info via context @@ -198,6 +290,7 @@ const char * nix_get_typename(nix_c_context * context, const nix_value * value); bool nix_get_bool(nix_c_context * context, const nix_value * value); /** @brief Get the raw string + * @ingroup value_extract * * This may contain placeholders. * @@ -205,21 +298,21 @@ bool nix_get_bool(nix_c_context * context, const nix_value * value); * @param[in] value Nix value to inspect * @param[in] callback Called with the string value. * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. - * @return string * @return error code, NIX_OK on success. */ nix_err nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return string, if the type is NIX_TYPE_PATH - * @return NULL in case of error. + * @return string valid while value is valid, NULL in case of error */ const char * nix_get_path_string(nix_c_context * context, const nix_value * value); /** @brief Get the length of a list + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return length of list, error info via context @@ -227,6 +320,7 @@ const char * nix_get_path_string(nix_c_context * context, const nix_value * valu unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); /** @brief Get the element count of an attrset + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return attrset element count, error info via context @@ -234,6 +328,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); /** @brief Get float value in 64 bits + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return float contents, error info via context @@ -241,6 +336,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value double nix_get_float(nix_c_context * context, const nix_value * value); /** @brief Get int value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return int contents, error info via context @@ -248,15 +344,18 @@ double nix_get_float(nix_c_context * context, const nix_value * value); int64_t nix_get_int(nix_c_context * context, const nix_value * value); /** @brief Get external reference + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return reference to external, NULL in case of error + * @return reference valid while value is valid. Call nix_gc_incref() if you need it to live longer, then only in that + * case call nix_gc_decref() when done. NULL in case of error */ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value); /** @brief Get the ix'th element of a list + * @ingroup value_extract * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -266,11 +365,12 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value); nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get the ix'th element of a list without forcing evaluation of the element + * @ingroup value_extract * * Returns the list element without forcing its evaluation, allowing access to lazy values. * The list value itself must already be evaluated. * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated list) * @param[in] state nix evaluator state @@ -281,8 +381,9 @@ nix_value * nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get an attr by name + * @ingroup value_extract * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -292,11 +393,12 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value + * @ingroup value_extract * * Returns the attribute value without forcing its evaluation, allowing access to lazy values. * The attribute set value itself must already be evaluated. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated attribute set) * @param[in] state nix evaluator state @@ -307,6 +409,7 @@ nix_value * nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Check if an attribute name exists on a value + * @ingroup value_extract * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -316,6 +419,7 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute by index + * @ingroup value_extract * * Also gives you the name. * @@ -329,18 +433,19 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state * @param[in] i attribute index - * @param[out] name will store a pointer to the attribute name + * @param[out] name will store a pointer to the attribute name, valid until state is freed * @return value, NULL in case of errors */ nix_value * nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute by index, without forcing evaluation of the attribute's value + * @ingroup value_extract * * Also gives you the name. * @@ -357,18 +462,19 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Use nix_gc_decref when you're done with the pointer + * Call nix_value_decref() when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect (must be an evaluated attribute set) * @param[in] state nix evaluator state * @param[in] i attribute index - * @param[out] name will store a pointer to the attribute name + * @param[out] name will store a pointer to the attribute name, valid until state is freed * @return value, NULL in case of errors */ nix_value * nix_get_attr_byidx_lazy( nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute name by index + * @ingroup value_extract * * Returns the attribute name without forcing evaluation of the attribute's value. * @@ -382,16 +488,14 @@ nix_value * nix_get_attr_byidx_lazy( * lexicographic order by Unicode scalar value for valid UTF-8). We recommend * applying this same ordering for consistency. * - * Owned by the nix EvalState * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state * @param[in] i attribute index - * @return name, NULL in case of errors + * @return name string valid until state is freed, NULL in case of errors */ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i); -/**@}*/ /** @name Initializers * * Values are typically "returned" by initializing already allocated memory that serves as the return value. @@ -401,6 +505,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, */ /**@{*/ /** @brief Set boolean value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] b the boolean value @@ -409,6 +514,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); /** @brief Set a string + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] str the string, copied @@ -417,6 +523,7 @@ nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); /** @brief Set a path + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] str the path string, copied @@ -425,6 +532,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); /** @brief Set a float + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] d the float, 64-bits @@ -433,6 +541,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); /** @brief Set an int + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] i the int @@ -441,6 +550,7 @@ nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); /** @brief Set null + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @return error code, NIX_OK on success. @@ -448,6 +558,7 @@ nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); nix_err nix_init_null(nix_c_context * context, nix_value * value); /** @brief Set the value to a thunk that will perform a function application when needed. + * @ingroup value_create * * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand. * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp. @@ -464,6 +575,7 @@ nix_err nix_init_null(nix_c_context * context, nix_value * value); nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); /** @brief Set an external value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] val the external value to set. Will be GC-referenced by the value. @@ -472,18 +584,25 @@ nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * f nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); /** @brief Create a list from a list builder + * @ingroup value_create + * + * After this call, the list builder becomes invalid and cannot be used again. + * The only necessary next step is to free it with nix_list_builder_free(). + * * @param[out] context Optional, stores error information - * @param[in] list_builder list builder to use. Make sure to unref this afterwards. + * @param[in] list_builder list builder to use * @param[out] value Nix value to modify * @return error code, NIX_OK on success. + * @see nix_list_builder_free */ nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); /** @brief Create a list builder + * @ingroup value_create * @param[out] context Optional, stores error information * @param[in] state nix evaluator state * @param[in] capacity how many bindings you'll add. Don't exceed. - * @return owned reference to a list builder. Make sure to unref when you're done. + * @return list builder. Call nix_list_builder_free() when you're done. */ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity); @@ -505,14 +624,21 @@ nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, uns void nix_list_builder_free(ListBuilder * list_builder); /** @brief Create an attribute set from a bindings builder + * @ingroup value_create + * + * After this call, the bindings builder becomes invalid and cannot be used again. + * The only necessary next step is to free it with nix_bindings_builder_free(). + * * @param[out] context Optional, stores error information * @param[out] value Nix value to modify - * @param[in] b bindings builder to use. Make sure to unref this afterwards. + * @param[in] b bindings builder to use * @return error code, NIX_OK on success. + * @see nix_bindings_builder_free */ nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); /** @brief Set primop + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] op primop, will be gc-referenced by the value @@ -521,6 +647,7 @@ nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuild */ nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); /** @brief Copy from another value + * @ingroup value_create * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] source value to copy from @@ -530,12 +657,11 @@ nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_val /**@}*/ /** @brief Create a bindings builder -* @param[out] context Optional, stores error information -* @param[in] state nix evaluator state -* @param[in] capacity how many bindings you'll add. Don't exceed. -* @return owned reference to a bindings builder. Make sure to unref when you're -done. -*/ + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @param[in] capacity how many bindings you'll add. Don't exceed. + * @return bindings builder. Call nix_bindings_builder_free() when you're done. + */ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity); /** @brief Insert bindings into a builder @@ -554,7 +680,6 @@ nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, * @param[in] builder the builder to free */ void nix_bindings_builder_free(BindingsBuilder * builder); -/**@}*/ /** @brief Realise a string context. * @@ -571,13 +696,13 @@ void nix_bindings_builder_free(BindingsBuilder * builder); * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false. You should set this to true when this call is part of a primop. You should set this to false when building for your application's purpose. - * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free + * @return NULL if failed, or a new nix_realised_string, which must be freed with nix_realised_string_free */ nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); /** @brief Start of the string * @param[in] realised_string - * @return pointer to the start of the string. It may not be null-terminated. + * @return pointer to the start of the string, valid until realised_string is freed. It may not be null-terminated. */ const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string); @@ -596,7 +721,7 @@ size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_s /** @brief Get a store path. The store paths are stored in an arbitrary order. * @param[in] realised_string * @param[in] index index of the store path, must be less than the count - * @return store path + * @return store path valid until realised_string is freed */ const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index); @@ -610,5 +735,4 @@ void nix_realised_string_free(nix_realised_string * realised_string); } #endif -/** @} */ #endif // NIX_API_VALUE_H diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 4d7f394fa..d301e5743 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -155,6 +155,8 @@ typedef struct nix_c_context nix_c_context; /** * @brief Called to get the value of a string owned by Nix. * + * The `start` data is borrowed and the function must not assume that the buffer persists after it returns. + * * @param[in] start the string to copy. * @param[in] n the string length. * @param[in] user_data optional, arbitrary data, passed to the nix_get_string_callback when it's called. From 883860c7ff6638f8069d8a6bb1be6ba2065c4608 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Tue, 28 Oct 2025 11:14:31 -0700 Subject: [PATCH 066/155] Move docker documentation to docker.io --- doc/manual/source/installation/installing-docker.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md index 9354c1a72..92fa55e1c 100644 --- a/doc/manual/source/installation/installing-docker.md +++ b/doc/manual/source/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti ghcr.io/nixos/nix -Unable to find image 'ghcr.io/nixos/nix:latest' locally -latest: Pulling from ghcr.io/nixos/nix +$ docker run -ti docker.io/nixos/nix +Unable to find image 'docker.io/nixos/nix:latest' locally +latest: Pulling from docker.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for ghcr.io/nixos/nix:latest +Status: Downloaded newer image for docker.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit From 943788754fc695dbe1b8cb3057f7fc1a16858e2c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Tue, 28 Oct 2025 11:16:37 -0700 Subject: [PATCH 067/155] Add ghcr for pre-release --- doc/manual/source/installation/installing-docker.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md index 92fa55e1c..ccc75be5a 100644 --- a/doc/manual/source/installation/installing-docker.md +++ b/doc/manual/source/installation/installing-docker.md @@ -16,6 +16,8 @@ nix (Nix) 2.3.12 35ca4ada6e96:/# exit ``` +> If you want the latest pre-release you can use ghcr.io/nixos/nix and view them at https://github.com/nixos/nix/pkgs/container/nix + # What is included in Nix's Docker image? The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` From f5aafbd6ed5ea7a38d27d51cf82d77634d341a05 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 28 Oct 2025 19:39:04 +0100 Subject: [PATCH 068/155] .coderabbit.yaml: Disable auto-review --- .coderabbit.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..5122f01e0 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,6 @@ +# Disable CodeRabbit auto-review to prevent verbose comments on PRs. +# When enabled: false, CodeRabbit won't attempt reviews and won't post +# "Review skipped" or other automated comments. +reviews: + auto_review: + enabled: false From e3246301a6dcd2c722241f4756484d40bc06f48a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 14:49:04 -0400 Subject: [PATCH 069/155] Enable JSON schema testing for derivation outputs I figured out what the problem was: the fragment needs to start with a `/`. --- src/json-schema-checks/meson.build | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 745fb5ffa..8e8ac57c4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -46,20 +46,19 @@ schemas = [ 'simple-derivation.json', ], }, - # # Not sure how to make subschema work - # { - # 'stem': 'derivation', - # 'schema': schema_dir / 'derivation-v3.yaml#output', - # 'files' : [ - # 'output-caFixedFlat.json', - # 'output-caFixedNAR.json', - # 'output-caFixedText.json', - # 'output-caFloating.json', - # 'output-deferred.json', - # 'output-impure.json', - # 'output-inputAddressed.json', - # ], - # }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output', + 'files' : [ + 'output-caFixedFlat.json', + 'output-caFixedNAR.json', + 'output-caFixedText.json', + 'output-caFloating.json', + 'output-deferred.json', + 'output-impure.json', + 'output-inputAddressed.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', From 84a5bee424ab25bd0dbc89b3abc6adb208142396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 28 Oct 2025 21:41:20 +0100 Subject: [PATCH 070/155] coderabbit: disable reporting review status --- .coderabbit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 5122f01e0..815dc27a5 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -4,3 +4,4 @@ reviews: auto_review: enabled: false + review_status: false From fe8cdbc3e41ecab02d451c8864e6309507d3c7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 28 Oct 2025 21:48:33 +0100 Subject: [PATCH 071/155] coderabbit: disable high_level_summary/poem/github status/sequence_diagrams --- .coderabbit.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 815dc27a5..00244700a 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -5,3 +5,10 @@ reviews: auto_review: enabled: false review_status: false + high_level_summary: false + poem: false + sequence_diagrams: false + changed_files_summary: false + tools: + github-checks: + enabled: false From be2572ed8d0c9dd626462229436ba7aaf2369690 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 17:16:38 -0400 Subject: [PATCH 072/155] Make `inputDrvs` JSON schema more precise It now captures the stable non-recursive format (just an output set) and the unstable recursive form for dynamic derivations. --- .../protocols/json/schema/derivation-v3.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index c950b839f..30fddf699 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -103,6 +103,13 @@ properties: > ``` > > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + additionalProperties: + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" system: type: string @@ -167,3 +174,28 @@ properties: title: Expected hash value description: | For fixed-output derivations, the expected content hash in base-16. + + outputName: + type: string + title: Output name + description: Name of the derivation output to depend on + + outputNames: + type: array + title: Output Names + description: Set of names of derivation outputs to depend on + items: + "$ref": "#/$defs/outputName" + + dynamicOutputs: + type: object + title: Dynamic Outputs + description: | + **Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) + + This recursive data type allows for depending on outputs of outputs. + properties: + outputs: + "$ref": "#/$defs/outputNames" + dynamicOutputs: + "$ref": "#/$defs/dynamicOutputs" From c67966418f99120a31e3d15c58a0aa253abfb151 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 16:59:35 -0400 Subject: [PATCH 073/155] Create JSON Schema for Store Paths We immediately use this in the JSON schemas for Derivation and Deriving Path, but we cannot yet use it in Store Object Info because those paths *do* include the store dir currently. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/derivation-v3.yaml | 20 ++++++------ .../json/schema/deriving-path-v1.yaml | 2 +- .../protocols/json/schema/store-path-v1 | 1 + .../protocols/json/schema/store-path-v1.yaml | 32 +++++++++++++++++++ .../source/protocols/json/store-path.md | 15 +++++++++ src/json-schema-checks/meson.build | 7 ++++ src/json-schema-checks/package.nix | 1 + src/json-schema-checks/store-path | 1 + 11 files changed, 72 insertions(+), 10 deletions(-) create mode 120000 doc/manual/source/protocols/json/schema/store-path-v1 create mode 100644 doc/manual/source/protocols/json/schema/store-path-v1.yaml create mode 100644 doc/manual/source/protocols/json/store-path.md create mode 120000 src/json-schema-checks/store-path diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 140fa9849..b7c9503ef 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -36,6 +36,7 @@ mkMesonDerivation (finalAttrs: { # For example JSON ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address + ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derived-path # Too many different types of files to filter for now ../../doc/manual diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index abd9422cd..7f3b1a103 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -119,6 +119,7 @@ - [JSON Formats](protocols/json/index.md) - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) + - [Store Path](protocols/json/store-path.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index f79667961..e8546d813 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -11,6 +11,7 @@ json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ 'hash-v1', 'content-address-v1', + 'store-path-v1', 'derivation-v3', 'deriving-path-v1', ] diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 30fddf699..3275bcdd9 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -85,7 +85,7 @@ properties: > ] > ``` items: - type: string + $ref: "store-path-v1.yaml" inputDrvs: type: object @@ -103,13 +103,15 @@ properties: > ``` > > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. - additionalProperties: - title: Store Path - description: | - A store path to a derivation, mapped to the outputs of that derivation. - oneOf: - - "$ref": "#/$defs/outputNames" - - "$ref": "#/$defs/dynamicOutputs" + patternProperties: + "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$": + title: Store Path + description: | + A store path to a derivation, mapped to the outputs of that derivation. + oneOf: + - "$ref": "#/$defs/outputNames" + - "$ref": "#/$defs/dynamicOutputs" + additionalProperties: false system: type: string @@ -155,7 +157,7 @@ properties: type: object properties: path: - type: string + $ref: "store-path-v1.yaml" title: Output path description: | The output path, if known in advance. diff --git a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml index 7fd74941e..11a784d06 100644 --- a/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/deriving-path-v1.yaml @@ -7,7 +7,7 @@ oneOf: - title: Constant description: | See [Constant](@docroot@/store/derivation/index.md#deriving-path-constant) deriving path. - type: string + $ref: "store-path-v1.yaml" - title: Output description: | See [Output](@docroot@/store/derivation/index.md#deriving-path-output) deriving path. diff --git a/doc/manual/source/protocols/json/schema/store-path-v1 b/doc/manual/source/protocols/json/schema/store-path-v1 new file mode 120000 index 000000000..31e7a6b2a --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-path-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/store-path \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml new file mode 100644 index 000000000..2012aab99 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -0,0 +1,32 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-path-v1.json" +title: Store Path +description: | + A [store path](@docroot@/store/store-path.md) identifying a store object. + + This schema describes the JSON representation of store paths as used in various Nix JSON APIs. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + + ## Format + + Store paths in JSON are represented as strings containing just the hash and name portion, without the store directory prefix. + + For example: `"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"` + + (If the store dir is `/nix/store`, then this corresponds to the path `/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv`.) + + ## Structure + + The format follows this pattern: `${digest}-${name}` + + - **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - **name**: The package name and optional version/suffix information + +type: string +pattern: "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$" +minLength: 34 diff --git a/doc/manual/source/protocols/json/store-path.md b/doc/manual/source/protocols/json/store-path.md new file mode 100644 index 000000000..02ecc8068 --- /dev/null +++ b/doc/manual/source/protocols/json/store-path.md @@ -0,0 +1,15 @@ +{{#include store-path-v1-fixed.md}} + +## Examples + +### Simple store path + +```json +{{#include schema/store-path-v1/simple.json}} +``` + + diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 745fb5ffa..f3e52e544 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -38,6 +38,13 @@ schemas = [ 'nar.json', ], }, + { + 'stem' : 'store-path', + 'schema' : schema_dir / 'store-path-v1.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'derivation', 'schema' : schema_dir / 'derivation-v3.yaml', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 6a76c8b28..0122b5493 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -22,6 +22,7 @@ mkMesonDerivation (finalAttrs: { ../../doc/manual/source/protocols/json/schema ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address + ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ./. diff --git a/src/json-schema-checks/store-path b/src/json-schema-checks/store-path new file mode 120000 index 000000000..003b1dbbb --- /dev/null +++ b/src/json-schema-checks/store-path @@ -0,0 +1 @@ +../../src/libstore-tests/data/store-path \ No newline at end of file From c874e7071b0f81406a4078e5ce0aec50770ccd53 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Oct 2025 01:47:18 +0300 Subject: [PATCH 074/155] libstore/http-binary-cache-store: Improve error messages in HttpBinaryCacheStore::upsertFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now the error message doesn't cram everything into a single line and we now instead get: error: … while uploading to HTTP binary cache at 's3://my-cache?endpoint=http://localhost:9000?compression%3Dzstd®ion=eu-west-1' error: unable to download 'http://localhost:9000/my-cache/nar/1125zqba8cx8wbfa632vy458a3j3xja0qpcqafsfdildyl9dqa7x.nar.xz': Operation was aborted by an application callback (42) --- src/libstore/http-binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 738db132d..1f9ee4100 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -157,7 +157,9 @@ void HttpBinaryCacheStore::upsertFile( try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg()); + UploadToHTTP err(e.message()); + err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); + throw err; } } From ae49074548bb3485a0a263ca862f6aee95cfb09f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Oct 2025 02:48:26 +0300 Subject: [PATCH 075/155] libstore/filetransfer: Add HttpMethod::PUT This got lost in f1968ea38e51201b37962a9cfd80775989a56d46 and now we had incorrect logs that confused "downloading" when we were in fact "uploading" things. --- src/libstore/filetransfer.cc | 4 +++- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/filetransfer.hh | 3 +++ src/libstore/s3-binary-cache-store.cc | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6b9c6602b..304984d99 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -394,9 +394,11 @@ struct curlFileTransfer : public FileTransfer if (request.method == HttpMethod::POST) { curl_easy_setopt(req, CURLOPT_POST, 1L); curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length()); - } else { + } else if (request.method == HttpMethod::PUT) { curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } else { + unreachable(); } curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 738db132d..089c7873a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upsertFile( uint64_t sizeHint) { auto req = makeRequest(path); - + req.method = HttpMethod::PUT; auto data = StreamToSourceAdapter(istream).drain(); auto compressionMethod = getCompressionMethod(path); diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 305c33af1..08a2b6329 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -88,6 +88,7 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT; */ enum struct HttpMethod { GET, + PUT, HEAD, POST, DELETE, @@ -147,7 +148,9 @@ struct FileTransferRequest case HttpMethod::HEAD: case HttpMethod::GET: return "download"; + case HttpMethod::PUT: case HttpMethod::POST: + assert(data); return "upload"; case HttpMethod::DELETE: return "delet"; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 828e75b7c..417355b68 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -101,6 +101,7 @@ std::string S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) { auto req = makeRequest(key); + req.method = HttpMethod::PUT; req.setupForS3(); auto url = req.uri.parsed(); From 6280905638aac9d15c09fc4d38aa469ee63d17be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 13:24:05 -0400 Subject: [PATCH 076/155] Convert store path info JSON docs to formal JSON Schema, and test This continues the work for formalizing our current JSON docs. Note that in the process, a few bugs were caught: - `closureSize` was repeated twice, forgot `closureDownloadSize` - `file*` fields should be `download*`. They are in fact called that in the line-oriented `.narinfo` file, but were renamed in the JSON format. --- doc/manual/package.nix | 2 + .../source/protocols/json/derivation.md | 2 +- doc/manual/source/protocols/json/hash.md | 2 +- doc/manual/source/protocols/json/meson.build | 1 + .../source/protocols/json/schema/nar-info-v1 | 1 + .../json/schema/store-object-info-v1 | 1 + .../json/schema/store-object-info-v1.yaml | 235 ++++++++++++++++++ .../protocols/json/store-object-info.md | 117 +++------ .../source/protocols/json/store-path.md | 2 +- src/json-schema-checks/meson.build | 50 ++++ src/json-schema-checks/nar-info | 1 + src/json-schema-checks/package.nix | 2 + src/json-schema-checks/store-object-info | 1 + 13 files changed, 327 insertions(+), 90 deletions(-) create mode 120000 doc/manual/source/protocols/json/schema/nar-info-v1 create mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v1 create mode 100644 doc/manual/source/protocols/json/schema/store-object-info-v1.yaml create mode 120000 src/json-schema-checks/nar-info create mode 120000 src/json-schema-checks/store-object-info diff --git a/doc/manual/package.nix b/doc/manual/package.nix index b7c9503ef..7d29df3c3 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -38,6 +38,8 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/derived-path + ../../src/libstore-tests/data/path-info + ../../src/libstore-tests/data/nar-info # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 602ab67e4..a4a4ea79d 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,6 +1,6 @@ {{#include derivation-v3-fixed.md}} - diff --git a/doc/manual/source/protocols/json/store-path.md b/doc/manual/source/protocols/json/store-path.md index 02ecc8068..cd18f6595 100644 --- a/doc/manual/source/protocols/json/store-path.md +++ b/doc/manual/source/protocols/json/store-path.md @@ -8,7 +8,7 @@ {{#include schema/store-path-v1/simple.json}} ``` - For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make. - If set to `0`, nix will detect the number of CPU cores and pass this number via NIX_BUILD_CORES. + If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`. > **Note** > diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index cb105a385..33ad8757a 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -87,7 +87,7 @@ struct AuthorizationSettings : Config {"*"}, "allowed-users", R"( - A list user names, separated by whitespace. + A list of user names, separated by whitespace. These users are allowed to connect to the Nix daemon. You can specify groups by prefixing names with `@`. From 4ea32d0b03f04143c54344363affea50fc804681 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 2 Nov 2025 14:00:07 +0100 Subject: [PATCH 108/155] Improve "resolution failed" error Previously: error: Cannot build '/nix/store/cqc798lwy2njwbdzgd0319z4r19j2d1w-nix-manual-2.33.0pre20251101_e4e4063.drv'. Reason: 1 dependency failed. Output paths: /nix/store/f1kln1c6z9r7rlhj0h9shcpch7j5g1fj-nix-manual-2.33.0pre20251101_e4e4063-man /nix/store/k65203rx5g1kcagpcz3c3a09bghcj92a-nix-manual-2.33.0pre20251101_e4e4063 error: Cannot build '/nix/store/ajk2fb6r7ijn2fc5c3h85n6zdi36xlfl-nixops-manual.drv'. Reason: 1 dependency failed. Output paths: /nix/store/0anr0998as8ry4hr5g3f3iarszx5aisx-nixops-manual error: resolution failed Now: error: Cannot build '/nix/store/cqc798lwy2njwbdzgd0319z4r19j2d1w-nix-manual-2.33.0pre20251101_e4e4063.drv'. Reason: 1 dependency failed. Output paths: /nix/store/f1kln1c6z9r7rlhj0h9shcpch7j5g1fj-nix-manual-2.33.0pre20251101_e4e4063-man /nix/store/k65203rx5g1kcagpcz3c3a09bghcj92a-nix-manual-2.33.0pre20251101_e4e4063 error: Cannot build '/nix/store/ajk2fb6r7ijn2fc5c3h85n6zdi36xlfl-nixops-manual.drv'. Reason: 1 dependency failed. Output paths: /nix/store/0anr0998as8ry4hr5g3f3iarszx5aisx-nixops-manual error: Build failed due to failed dependency --- src/libstore/build/derivation-goal.cc | 2 +- tests/functional/build.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 717d6890a..14aa044ea 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -147,7 +147,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); } if (nrFailed != 0) { - co_return doneFailure({BuildResult::Failure::DependencyFailed, "resolution failed"}); + co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); } if (resolutionGoal->resolvedDrv) { diff --git a/tests/functional/build.sh b/tests/functional/build.sh index c9a39438d..0b06dcd91 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -184,6 +184,7 @@ test "$status" = 1 if isDaemonNewer "2.29pre"; then <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" <<<"$out" grepQuiet -E "Reason: 1 dependency failed." + <<<"$out" grepQuiet -E "Build failed due to failed dependency" else <<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" fi From 233bd250d175719896ef4985acb4a41613cb34c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 2 Nov 2025 14:10:12 +0100 Subject: [PATCH 109/155] flake: Update, nixos-25.05-small -> nixos-25.05 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d98ce345cdab58477ca61855540999c86577d19d?narHash=sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8%3D' (2025-08-26) → 'github:NixOS/nixpkgs/daf6dc47aa4b44791372d6139ab7b25269184d55?narHash=sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8%2BON/0Yy8%2Ba5vsDU%3D' (2025-10-27) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index cc2b2f27e..63290ef86 100644 --- a/flake.lock +++ b/flake.lock @@ -63,16 +63,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1756178832, - "narHash": "sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8=", + "lastModified": 1761597516, + "narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d98ce345cdab58477ca61855540999c86577d19d", + "rev": "daf6dc47aa4b44791372d6139ab7b25269184d55", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-25.05-small", + "ref": "nixos-25.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 418f3180f..e25722d46 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; From bf947bfc26704b3a21da222f3c67fb9d773383b9 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 21 Oct 2025 06:19:17 +0000 Subject: [PATCH 110/155] feat(libstore/s3-binary-cache-store): add multipart upload config settings Add three configuration settings to `S3BinaryCacheStoreConfig` to control multipart upload behavior: - `bool multipart-upload` (default `false`): Enable/disable multipart uploads - `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part - `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart The feature is disabled by default. --- .../nix/store/s3-binary-cache-store.hh | 32 +++++++++++++++++++ src/libstore/s3-binary-cache-store.cc | 23 +++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 81a2d3f3f..bf86d0671 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; + const Setting multipartUpload{ + this, + false, + "multipart-upload", + R"( + Whether to use multipart uploads for large files. When enabled, + files exceeding the multipart threshold will be uploaded in + multiple parts, which is required for files larger than 5 GiB and + can improve performance and reliability for large uploads. + )"}; + + const Setting multipartChunkSize{ + this, + 5 * 1024 * 1024, + "multipart-chunk-size", + R"( + The size (in bytes) of each part in multipart uploads. Must be + at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the + number of requests but use more memory. Default is 5 MiB. + )", + {"buffer-size"}}; + + const Setting multipartThreshold{ + this, + 100 * 1024 * 1024, + "multipart-threshold", + R"( + The minimum file size (in bytes) for using multipart uploads. + Files smaller than this threshold will use regular PUT requests. + Default is 100 MiB. Only takes effect when multipart-upload is enabled. + )"}; + /** * Set of settings that are part of the S3 URI itself. * These are needed for region specification and other S3-specific settings. diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 9303a80f8..4cf5f987a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -15,6 +15,7 @@ namespace nix { MakeError(UploadToS3, Error); +static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB class S3BinaryCacheStore : public virtual HttpBinaryCacheStore @@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( cacheUri.query[key] = value; } } + + if (multipartChunkSize < AWS_MIN_PART_SIZE) { + throw UsageError( + "multipart-chunk-size must be at least %s, got %s", + renderSize(AWS_MIN_PART_SIZE), + renderSize(multipartChunkSize.get())); + } + + if (multipartChunkSize > AWS_MAX_PART_SIZE) { + throw UsageError( + "multipart-chunk-size must be at most %s, got %s", + renderSize(AWS_MAX_PART_SIZE), + renderSize(multipartChunkSize.get())); + } + + if (multipartUpload && multipartThreshold < multipartChunkSize) { + warn( + "multipart-threshold (%s) is less than multipart-chunk-size (%s), " + "which may result in single-part multipart uploads", + renderSize(multipartThreshold.get()), + renderSize(multipartChunkSize.get())); + } } std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const From 040d1aae41a3bfda86c29910eb1495d75598fd35 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 22 Oct 2025 08:42:32 +0000 Subject: [PATCH 111/155] feat(libstore/s3-binary-cache-store): implement `uploadMultipart()` Implement `uploadMultipart()`, the main method that orchestrates S3 multipart uploads --- src/libstore/s3-binary-cache-store.cc | 228 ++++++++++++++++++++++---- 1 file changed, 192 insertions(+), 36 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4cf5f987a..37264dfae 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,7 @@ #include "nix/util/util.hh" #include +#include #include #include #include @@ -17,6 +18,7 @@ MakeError(UploadToS3, Error); static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB +static constexpr uint64_t AWS_MAX_PART_COUNT = 10000; class S3BinaryCacheStore : public virtual HttpBinaryCacheStore { @@ -51,9 +53,48 @@ private: std::optional contentEncoding); /** - * Uploads a file to S3 (CompressedSource overload). + * Uploads a file to S3 using multipart upload. + * + * This method is suitable for large files that exceed the multipart threshold. + * It orchestrates the complete multipart upload process: creating the upload, + * splitting the data into parts, uploading each part, and completing the upload. + * If any error occurs, the multipart upload is automatically aborted. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html */ - void upload(std::string_view path, CompressedSource & source, std::string_view mimeType); + void uploadMultipart( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding); + + /** + * A Sink that manages a complete S3 multipart upload lifecycle. + * Creates the upload on construction, buffers and uploads chunks as data arrives, + * and completes or aborts the upload appropriately. + */ + struct MultipartSink : Sink + { + S3BinaryCacheStore & store; + std::string_view path; + std::string uploadId; + std::string::size_type chunkSize; + + std::vector partEtags; + std::string buffer; + + MultipartSink( + S3BinaryCacheStore & store, + std::string_view path, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding); + + void operator()(std::string_view data) override; + void finish(); + void uploadChunk(std::string chunk); + }; /** * Creates a multipart upload for large objects to S3. @@ -73,18 +114,13 @@ private: */ std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data); - struct UploadedPart - { - uint64_t partNumber; - std::string etag; - }; - /** * Completes a multipart upload by combining all uploaded parts. * @see * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_RequestSyntax */ - void completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span parts); + void + completeMultipartUpload(std::string_view key, std::string_view uploadId, std::span partEtags); /** * Abort a multipart upload @@ -92,17 +128,31 @@ private: * @see * https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax */ - void abortMultipartUpload(std::string_view key, std::string_view uploadId); + void abortMultipartUpload(std::string_view key, std::string_view uploadId) noexcept; }; void S3BinaryCacheStore::upsertFile( const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) { - if (auto compressionMethod = getCompressionMethod(path)) { - CompressedSource compressed(source, *compressionMethod); - upload(path, compressed, mimeType); - } else { - upload(path, source, sizeHint, mimeType, std::nullopt); + auto doUpload = [&](RestartableSource & src, uint64_t size, std::optional encoding) { + if (s3Config->multipartUpload && size > s3Config->multipartThreshold) { + uploadMultipart(path, src, size, mimeType, encoding); + } else { + upload(path, src, size, mimeType, encoding); + } + }; + + try { + if (auto compressionMethod = getCompressionMethod(path)) { + CompressedSource compressed(source, *compressionMethod); + doUpload(compressed, compressed.size(), compressed.getCompressionMethod()); + } else { + doUpload(source, sizeHint, std::nullopt); + } + } catch (FileTransferError & e) { + UploadToS3 err(e.message()); + err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); + throw err; } } @@ -120,18 +170,112 @@ void S3BinaryCacheStore::upload( renderSize(sizeHint), renderSize(AWS_MAX_PART_SIZE)); - try { - HttpBinaryCacheStore::upload(path, source, sizeHint, mimeType, contentEncoding); - } catch (FileTransferError & e) { - UploadToS3 err(e.message()); - err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); - throw err; + HttpBinaryCacheStore::upload(path, source, sizeHint, mimeType, contentEncoding); +} + +void S3BinaryCacheStore::uploadMultipart( + std::string_view path, + RestartableSource & source, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding) +{ + debug("using S3 multipart upload for '%s' (%d bytes)", path, sizeHint); + MultipartSink sink(*this, path, sizeHint, mimeType, contentEncoding); + source.drainInto(sink); + sink.finish(); +} + +S3BinaryCacheStore::MultipartSink::MultipartSink( + S3BinaryCacheStore & store, + std::string_view path, + uint64_t sizeHint, + std::string_view mimeType, + std::optional contentEncoding) + : store(store) + , path(path) +{ + // Calculate chunk size and estimated parts + chunkSize = store.s3Config->multipartChunkSize; + uint64_t estimatedParts = (sizeHint + chunkSize - 1) / chunkSize; // ceil division + + if (estimatedParts > AWS_MAX_PART_COUNT) { + // Equivalent to ceil(sizeHint / AWS_MAX_PART_COUNT) + uint64_t minChunkSize = (sizeHint + AWS_MAX_PART_COUNT - 1) / AWS_MAX_PART_COUNT; + + if (minChunkSize > AWS_MAX_PART_SIZE) { + throw Error( + "file too large for S3 multipart upload: %s would require chunk size of %s " + "(max %s) to stay within %d part limit", + renderSize(sizeHint), + renderSize(minChunkSize), + renderSize(AWS_MAX_PART_SIZE), + AWS_MAX_PART_COUNT); + } + + warn( + "adjusting S3 multipart chunk size from %s to %s " + "to stay within %d part limit for %s file", + renderSize(store.s3Config->multipartChunkSize.get()), + renderSize(minChunkSize), + AWS_MAX_PART_COUNT, + renderSize(sizeHint)); + + chunkSize = minChunkSize; + estimatedParts = AWS_MAX_PART_COUNT; + } + + buffer.reserve(chunkSize); + partEtags.reserve(estimatedParts); + uploadId = store.createMultipartUpload(path, mimeType, contentEncoding); +} + +void S3BinaryCacheStore::MultipartSink::operator()(std::string_view data) +{ + buffer.append(data); + + while (buffer.size() >= chunkSize) { + // Move entire buffer, extract excess, copy back remainder + auto chunk = std::move(buffer); + auto excessSize = chunk.size() > chunkSize ? chunk.size() - chunkSize : 0; + if (excessSize > 0) { + buffer.resize(excessSize); + std::memcpy(buffer.data(), chunk.data() + chunkSize, excessSize); + } + chunk.resize(std::min(chunkSize, chunk.size())); + uploadChunk(std::move(chunk)); } } -void S3BinaryCacheStore::upload(std::string_view path, CompressedSource & source, std::string_view mimeType) +void S3BinaryCacheStore::MultipartSink::finish() { - upload(path, static_cast(source), source.size(), mimeType, source.getCompressionMethod()); + if (!buffer.empty()) { + uploadChunk(std::move(buffer)); + } + + try { + if (partEtags.empty()) { + throw Error("no data read from stream"); + } + store.completeMultipartUpload(path, uploadId, partEtags); + } catch (Error & e) { + store.abortMultipartUpload(path, uploadId); + e.addTrace({}, "while finishing an S3 multipart upload"); + throw; + } +} + +void S3BinaryCacheStore::MultipartSink::uploadChunk(std::string chunk) +{ + auto partNumber = partEtags.size() + 1; + try { + std::string etag = store.uploadPart(path, uploadId, partNumber, std::move(chunk)); + partEtags.push_back(std::move(etag)); + } catch (Error & e) { + store.abortMultipartUpload(path, uploadId); + e.addTrace({}, "while uploading part %d of an S3 multipart upload", partNumber); + throw; + } } std::string S3BinaryCacheStore::createMultipartUpload( @@ -171,6 +315,10 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::string S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data) { + if (partNumber > AWS_MAX_PART_COUNT) { + throw Error("S3 multipart upload exceeded %d part limit", AWS_MAX_PART_COUNT); + } + auto req = makeRequest(key); req.method = HttpMethod::PUT; req.setupForS3(); @@ -189,24 +337,29 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, throw Error("S3 UploadPart response missing ETag for part %d", partNumber); } + debug("Part %d uploaded, ETag: %s", partNumber, result.etag); return std::move(result.etag); } -void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) +void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId) noexcept { - auto req = makeRequest(key); - req.setupForS3(); + try { + auto req = makeRequest(key); + req.setupForS3(); - auto url = req.uri.parsed(); - url.query["uploadId"] = uploadId; - req.uri = VerbatimURL(url); - req.method = HttpMethod::DELETE; + auto url = req.uri.parsed(); + url.query["uploadId"] = uploadId; + req.uri = VerbatimURL(url); + req.method = HttpMethod::DELETE; - getFileTransfer()->enqueueFileTransfer(req).get(); + getFileTransfer()->enqueueFileTransfer(req).get(); + } catch (...) { + ignoreExceptionInDestructor(); + } } void S3BinaryCacheStore::completeMultipartUpload( - std::string_view key, std::string_view uploadId, std::span parts) + std::string_view key, std::string_view uploadId, std::span partEtags) { auto req = makeRequest(key); req.setupForS3(); @@ -217,21 +370,24 @@ void S3BinaryCacheStore::completeMultipartUpload( req.method = HttpMethod::POST; std::string xml = ""; - for (const auto & part : parts) { + for (const auto & [idx, etag] : enumerate(partEtags)) { xml += ""; - xml += "" + std::to_string(part.partNumber) + ""; - xml += "" + part.etag + ""; + // S3 part numbers are 1-indexed, but vector indices are 0-indexed + xml += "" + std::to_string(idx + 1) + ""; + xml += "" + etag + ""; xml += ""; } xml += ""; - debug("S3 CompleteMultipartUpload XML (%d parts): %s", parts.size(), xml); + debug("S3 CompleteMultipartUpload XML (%d parts): %s", partEtags.size(), xml); StringSource payload{xml}; req.data = {payload}; req.mimeType = "text/xml"; getFileTransfer()->enqueueFileTransfer(req).get(); + + debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } StringSet S3BinaryCacheStoreConfig::uriSchemes() From 965d6be7c1962b87d47eb229153e2b5685c64739 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Thu, 30 Oct 2025 19:22:06 +0000 Subject: [PATCH 112/155] tests(nixos/s3-binary-cache-store): enable multipart --- tests/nixos/s3-binary-cache-store.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a07375489..a2ba1dae6 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -794,10 +794,9 @@ in test_compression_disabled() test_nix_prefetch_url() test_versioned_urls() - # FIXME: enable when multipart fully lands - # test_multipart_upload_basic() - # test_multipart_threshold() - # test_multipart_with_log_compression() + test_multipart_upload_basic() + test_multipart_threshold() + test_multipart_with_log_compression() print("\n" + "="*80) print("✓ All S3 Binary Cache Store Tests Passed!") From 3448d4fa4c7fca0d62487fa6ac0dfded72ff18de Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sat, 25 Oct 2025 05:05:10 +0000 Subject: [PATCH 113/155] docs(rl-next/s3-curl-implementation): update with multipart uploads --- doc/manual/rl-next/s3-curl-implementation.md | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/doc/manual/rl-next/s3-curl-implementation.md b/doc/manual/rl-next/s3-curl-implementation.md index fab010010..2647ac581 100644 --- a/doc/manual/rl-next/s3-curl-implementation.md +++ b/doc/manual/rl-next/s3-curl-implementation.md @@ -1,6 +1,6 @@ --- synopsis: "Improved S3 binary cache support via HTTP" -prs: [13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 13752] +prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421] issues: [13084, 12671, 11748, 12403] --- @@ -18,9 +18,23 @@ improvements: The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management. -All existing S3 URL formats and parameters remain supported, with the notable -exception of multi-part uploads, which are no longer supported. +All existing S3 URL formats and parameters remain supported, however the store +settings for configuring multipart uploads have changed: + +- **`multipart-upload`** (default: `false`): Enable multipart uploads for large + files. When enabled, files exceeding the multipart threshold will be uploaded + in multiple parts. + +- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using + multipart uploads. Files smaller than this will use regular PUT requests. + Only takes effect when `multipart-upload` is enabled. + +- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart + uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes + reduce the number of requests but use more memory. + +- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it. Note that this change also means Nix now supports S3 binary cache stores even -if build without `aws-crt-cpp`, but only for public buckets which do not -require auth. +if built without `aws-crt-cpp`, but only for public buckets which do not +require authentication. From 81a2809a526e4fcc887d3178c8b48646320a25e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Nov 2025 12:01:55 +0100 Subject: [PATCH 114/155] Apply updated nixfmt --- doc/manual/generate-store-types.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix index a03d3d621..4e06c7f60 100644 --- a/doc/manual/generate-store-types.nix +++ b/doc/manual/generate-store-types.nix @@ -24,9 +24,9 @@ let in concatStringsSep "\n" (map showEntry storesList); - "index.md" = - replaceStrings [ "@store-types@" ] [ index ] - (readFile ./source/store/types/index.md.in); + "index.md" = replaceStrings [ "@store-types@" ] [ index ] ( + readFile ./source/store/types/index.md.in + ); tableOfContents = let From bd420928730cf268ffe33071292e044118a0c57c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Nov 2025 16:54:22 -0400 Subject: [PATCH 115/155] Use less `c_str()` in the evaluator, and other cleanups It is better to avoid null termination for performance and memory safety, wherever possible. These are good cleanups extracted from the Pascal String work that we can land by themselves first, shrinking the diff in that PR. Co-Authored-By: Aspen Smith Co-Authored-By: Sergei Zimmerman --- src/libexpr-c/nix_api_value.cc | 2 +- .../include/nix/expr/tests/libexpr.hh | 2 +- src/libexpr-tests/value/value.cc | 18 +++++++++++++++ src/libexpr/eval-cache.cc | 6 ++--- src/libexpr/eval.cc | 23 +++++++++++-------- src/libexpr/get-drvs.cc | 8 +++---- src/libexpr/include/nix/expr/get-drvs.hh | 2 +- src/libexpr/include/nix/expr/value.hh | 7 +++++- src/libexpr/nixexpr.cc | 2 +- src/libexpr/primops.cc | 6 ++--- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 ++--- src/libflake/flake.cc | 6 ++--- src/libutil-c/nix_api_util.cc | 4 ++-- src/libutil-c/nix_api_util_internal.h | 2 +- src/nix/nix-env/nix-env.cc | 6 ++--- 16 files changed, 64 insertions(+), 38 deletions(-) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 3b8c7dd04..e231c36f4 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -235,7 +235,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_ try { auto & v = check_value_in(value); assert(v.type() == nix::nString); - call_nix_get_string_callback(v.c_str(), callback, user_data); + call_nix_get_string_callback(v.string_view(), callback, user_data); } NIXC_CATCH_ERRS } diff --git a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh index a1320e14a..daae00802 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh @@ -106,7 +106,7 @@ MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s)) if (arg.type() != nString) { return false; } - return std::string_view(arg.c_str()) == s; + return arg.string_view() == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) diff --git a/src/libexpr-tests/value/value.cc b/src/libexpr-tests/value/value.cc index 63501dd49..229e449db 100644 --- a/src/libexpr-tests/value/value.cc +++ b/src/libexpr-tests/value/value.cc @@ -1,6 +1,7 @@ #include "nix/expr/value.hh" #include "nix/store/tests/libstore.hh" +#include namespace nix { @@ -22,4 +23,21 @@ TEST_F(ValueTest, vInt) ASSERT_EQ(true, vInt.isValid()); } +TEST_F(ValueTest, staticString) +{ + Value vStr1; + Value vStr2; + vStr1.mkStringNoCopy("foo"); + vStr2.mkStringNoCopy("foo"); + + auto sd1 = vStr1.string_view(); + auto sd2 = vStr2.string_view(); + + // The strings should be the same + ASSERT_EQ(sd1, sd2); + + // The strings should also be backed by the same (static) allocation + ASSERT_EQ(sd1.data(), sd2.data()); +} + } // namespace nix diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 480ca72c7..de74d2143 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -406,7 +406,7 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), string_t{v.c_str(), {}}}; + cachedValue = {root->db->setString(getKey(), v.string_view(), v.context()), string_t{v.string_view(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -541,7 +541,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow(); - return v.type() == nString ? v.c_str() : v.path().to_string(); + return v.type() == nString ? std::string(v.string_view()) : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -580,7 +580,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.c_str(), std::move(context)}; + return {std::string{v.string_view()}, std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 873b88986..e2687148b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2366,12 +2366,15 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { - return !canonicalizePath && !copyToStore - ? // FIXME: hack to preserve path literals that end in a - // slash, as in /foo/${x}. - v.pathStr() - : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) - : std::string(v.path().path.abs()); + if (!canonicalizePath && !copyToStore) { + // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + return v.pathStrView(); + } else if (copyToStore) { + return store->printStorePath(copyPathToStore(context, v.path())); + } else { + return std::string{v.path().path.abs()}; + } } if (v.type() == nAttrs) { @@ -2624,7 +2627,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st return; case nString: - if (strcmp(v1.c_str(), v2.c_str()) != 0) { + if (v1.string_view() != v2.string_view()) { error( "string '%s' is not equal to string '%s'", ValuePrinter(*this, v1, errorPrintOptions), @@ -2641,7 +2644,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st ValuePrinter(*this, v2, errorPrintOptions)) .debugThrow(); } - if (strcmp(v1.pathStr(), v2.pathStr()) != 0) { + if (v1.pathStrView() != v2.pathStrView()) { error( "path '%s' is not equal to path '%s'", ValuePrinter(*this, v1, errorPrintOptions), @@ -2807,12 +2810,12 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean() == v2.boolean(); case nString: - return strcmp(v1.c_str(), v2.c_str()) == 0; + return v1.string_view() == v2.string_view(); case nPath: return // FIXME: compare accessors by their fingerprint. - v1.pathAccessor() == v2.pathAccessor() && strcmp(v1.pathStr(), v2.pathStr()) == 0; + v1.pathAccessor() == v2.pathAccessor() && v1.pathStrView() == v2.pathStrView(); case nNull: return true; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 5a7281b2b..c4a2b00af 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -168,7 +168,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT for (auto elem : outTI->listView()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->c_str()); + auto out = outputs.find(elem->string_view()); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -245,7 +245,7 @@ std::string PackageInfo::queryMetaString(const std::string & name) Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->c_str(); + return std::string{v->string_view()}; } NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) @@ -258,7 +258,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) + if (auto n = string2Int(v->string_view())) return NixInt{*n}; } return def; @@ -274,7 +274,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->c_str())) + if (auto n = string2Float(v->string_view())) return *n; } return def; diff --git a/src/libexpr/include/nix/expr/get-drvs.hh b/src/libexpr/include/nix/expr/get-drvs.hh index 3d42188bf..4beccabe2 100644 --- a/src/libexpr/include/nix/expr/get-drvs.hh +++ b/src/libexpr/include/nix/expr/get-drvs.hh @@ -15,7 +15,7 @@ namespace nix { struct PackageInfo { public: - typedef std::map> Outputs; + typedef std::map, std::less<>> Outputs; private: EvalState * state; diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 22d85dc99..706a4fe3f 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -1109,7 +1109,7 @@ public: std::string_view string_view() const noexcept { - return std::string_view(getStorage().c_str); + return std::string_view{getStorage().c_str}; } const char * c_str() const noexcept @@ -1177,6 +1177,11 @@ public: return getStorage().path; } + std::string_view pathStrView() const noexcept + { + return std::string_view{getStorage().path}; + } + SourceAccessor * pathAccessor() const noexcept { return getStorage().accessor; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index b183f1bbf..a1d1b7e4b 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const { - str << v.pathStr(); + str << v.pathStrView(); } void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 04196bc1f..96e79fedd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -691,12 +691,12 @@ struct CompareValues case nFloat: return v1->fpoint() < v2->fpoint(); case nString: - return strcmp(v1->c_str(), v2->c_str()) < 0; + return v1->string_view() < v2->string_view(); case nPath: // Note: we don't take the accessor into account // since it's not obvious how to compare them in a // reproducible way. - return strcmp(v1->pathStr(), v2->pathStr()) < 0; + return v1->pathStrView() < v2->pathStrView(); case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -2930,7 +2930,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value ** args, V for (const auto & [n, i] : enumerate(*args[0]->attrs())) list[n] = Value::toPtr(state.symbols[i.name]); - std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; }); + std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return v1->string_view() < v2->string_view(); }); v.mkList(list); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 2cd853f60..03b14b83c 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -33,7 +33,7 @@ json printValueAsJSON( case nString: copyContext(v, context); - out = v.c_str(); + out = v.string_view(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index d5959e894..0a7a334f4 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -82,7 +82,7 @@ static void printValueAsXML( case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); + doc.writeEmptyElement("string", singletonAttrs("value", v.string_view())); break; case nPath: @@ -102,14 +102,14 @@ static void printValueAsXML( if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->c_str(); + xmlAttrs["drvPath"] = drvPath = a->value->string_view(); } if (auto a = v.attrs()->get(state.s.outPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->c_str(); + xmlAttrs["outPath"] = a->value->string_view(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 42385712c..dc60dbf08 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -97,7 +97,7 @@ static void parseFlakeInputAttr(EvalState & state, const Attr & attr, fetchers:: #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->c_str()); + attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view())); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); @@ -177,7 +177,7 @@ static FlakeInput parseFlakeInput( parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputAttrPath(attr.value->c_str())); + auto follows(parseInputAttrPath(attr.value->string_view())); follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end()); input.follows = follows; } else @@ -264,7 +264,7 @@ static Flake readFlake( if (auto description = vInfo.attrs()->get(state.s.description)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->c_str(); + flake.description = description->value->string_view(); } auto sInputs = state.symbols.create("inputs"); diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 3903823aa..5934e8479 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -153,9 +153,9 @@ nix_err nix_err_code(const nix_c_context * read_context) } // internal -nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data) +nix_err call_nix_get_string_callback(const std::string_view str, nix_get_string_callback callback, void * user_data) { - callback(str.c_str(), str.size(), user_data); + callback(str.data(), str.size(), user_data); return NIX_OK; } diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index 92bb9c1d2..e4c5e93bb 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -32,7 +32,7 @@ nix_err nix_context_error(nix_c_context * context); * @return NIX_OK if there were no errors. * @see nix_get_string_callback */ -nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data); +nix_err call_nix_get_string_callback(const std::string_view str, nix_get_string_callback callback, void * user_data); #define NIXC_CATCH_ERRS \ catch (...) \ diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 01c8ccf4b..2a0984d18 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->c_str(); + attrs2["value"] = v->string_view(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1249,7 +1249,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->c_str(); + attrs3["value"] = elem->string_view(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = i.value->c_str(); + attrs3["value"] = i.value->string_view(); xml.writeEmptyElement("string", attrs3); } } From 2f6c865e25ee41ec2ba5b8f087a29512ad7aff82 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Nov 2025 13:22:28 +0100 Subject: [PATCH 116/155] getAccessorFromCommit(): Remove superfluous infoAttrs variable --- src/libfetchers/git.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c8311c17f..710d2f315 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -735,13 +735,10 @@ struct GitInputScheme : InputScheme auto rev = *input.getRev(); - Attrs infoAttrs({ - {"rev", rev.gitRev()}, - {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)}, - }); + input.attrs.insert_or_assign("lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)); if (!getShallowAttr(input)) - infoAttrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev)); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -797,9 +794,6 @@ struct GitInputScheme : InputScheme } assert(!origRev || origRev == rev); - if (!getShallowAttr(input)) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return {accessor, std::move(input)}; } From 4a0ccc89d9721fd41dc66f74b475f39df60ed20f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Nov 2025 13:58:23 +0100 Subject: [PATCH 117/155] ThreadPool::enqueue(): Use move semantics This avoids a superfluous copy of the work item. --- src/libutil/include/nix/util/thread-pool.hh | 2 +- src/libutil/thread-pool.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index 811c03d88..a07354146 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -36,7 +36,7 @@ public: /** * Enqueue a function to be executed by the thread pool. */ - void enqueue(const work_t & t); + void enqueue(work_t t); /** * Execute work items until the queue is empty. diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index b7740bc3e..24bdeef86 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -41,12 +41,12 @@ void ThreadPool::shutdown() thr.join(); } -void ThreadPool::enqueue(const work_t & t) +void ThreadPool::enqueue(work_t t) { auto state(state_.lock()); if (quit) throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); - state->pending.push(t); + state->pending.push(std::move(t)); /* Note: process() also executes items, so count it as a worker. */ if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) state->workers.emplace_back(&ThreadPool::doWork, this, false); From 53b4ea6c85e2d000b8badc923066866ba8de324c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 27 Oct 2025 11:26:46 -0700 Subject: [PATCH 118/155] Add documentation for NAR spec in kaitai * Add a new flake check * Add unit tests * Add Kaitai spec * Updated documentation --- doc/manual/source/SUMMARY.md.in | 2 +- .../{nix-archive.md => nix-archive/index.md} | 12 ++ .../source/protocols/nix-archive/nar.ksy | 169 ++++++++++++++++++ .../file-system-object/content-address.md | 2 +- flake.nix | 4 + meson.build | 1 + packaging/components.nix | 5 + packaging/dev-shell.nix | 4 +- packaging/hydra.nix | 1 + src/kaitai-struct-checks/.version | 1 + src/kaitai-struct-checks/meson.build | 77 ++++++++ src/kaitai-struct-checks/nar.ksy | 1 + src/kaitai-struct-checks/nars | 1 + .../nix-meson-build-support | 1 + src/kaitai-struct-checks/package.nix | 75 ++++++++ src/kaitai-struct-checks/test-parse-nar.cc | 48 +++++ src/nix/nar.md | 2 +- 17 files changed, 402 insertions(+), 4 deletions(-) rename doc/manual/source/protocols/{nix-archive.md => nix-archive/index.md} (73%) create mode 100644 doc/manual/source/protocols/nix-archive/nar.ksy create mode 120000 src/kaitai-struct-checks/.version create mode 100644 src/kaitai-struct-checks/meson.build create mode 120000 src/kaitai-struct-checks/nar.ksy create mode 120000 src/kaitai-struct-checks/nars create mode 120000 src/kaitai-struct-checks/nix-meson-build-support create mode 100644 src/kaitai-struct-checks/package.nix create mode 100644 src/kaitai-struct-checks/test-parse-nar.cc diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 7f3b1a103..287dff872 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -125,7 +125,7 @@ - [Deriving Path](protocols/json/deriving-path.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - - [Nix Archive (NAR) Format](protocols/nix-archive.md) + - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) diff --git a/doc/manual/source/protocols/nix-archive.md b/doc/manual/source/protocols/nix-archive/index.md similarity index 73% rename from doc/manual/source/protocols/nix-archive.md rename to doc/manual/source/protocols/nix-archive/index.md index 02a8dd464..4d25f63e2 100644 --- a/doc/manual/source/protocols/nix-archive.md +++ b/doc/manual/source/protocols/nix-archive/index.md @@ -41,3 +41,15 @@ The `str` function / parameterized rule is defined as follows: - `int(n)` = the 64-bit little endian representation of the number `n` - `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte + +## Kaitai Struct Specification + +The Nix Archive (NAR) format is also formally described using [Kaitai Struct](https://kaitai.io/), an Interface Description Language (IDL) for defining binary data structures. + +> Kaitai Struct provides a language-agnostic, machine-readable specification that can be compiled into parsers for various programming languages (e.g., C++, Python, Java, Rust). + +```yaml +{{#include nar.ksy}} +``` + +The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed. \ No newline at end of file diff --git a/doc/manual/source/protocols/nix-archive/nar.ksy b/doc/manual/source/protocols/nix-archive/nar.ksy new file mode 100644 index 000000000..1cad09097 --- /dev/null +++ b/doc/manual/source/protocols/nix-archive/nar.ksy @@ -0,0 +1,169 @@ +meta: + id: nix_nar + title: Nix Archive (NAR) + file-extension: nar + endian: le +doc: | + Nix Archive (NAR) format. A simple, reproducible binary archive + format used by the Nix package manager to serialize file system objects. +doc-ref: 'https://nixos.org/manual/nix/stable/command-ref/nix-store.html#nar-format' + +seq: + - id: magic + type: padded_str + doc: "Magic string, must be 'nix-archive-1'." + valid: + expr: _.body == 'nix-archive-1' + - id: root_node + type: node + doc: "The root of the archive, which is always a single node." + +types: + padded_str: + doc: | + A string, prefixed with its length (u8le) and + padded with null bytes to the next 8-byte boundary. + seq: + - id: len_str + type: u8 + - id: body + type: str + size: len_str + encoding: 'ascii' + - id: padding + size: (8 - (len_str % 8)) % 8 + + node: + doc: "A single filesystem node (file, directory, or symlink)." + seq: + - id: open_paren + type: padded_str + doc: "Must be '(', a token starting the node definition." + valid: + expr: _.body == '(' + - id: type_key + type: padded_str + doc: "Must be 'type'." + valid: + expr: _.body == 'type' + - id: type_val + type: padded_str + doc: "The type of the node: 'regular', 'directory', or 'symlink'." + - id: body + type: + switch-on: type_val.body + cases: + "'directory'": type_directory + "'regular'": type_regular + "'symlink'": type_symlink + - id: close_paren + type: padded_str + valid: + expr: _.body == ')' + if: "type_val.body != 'directory'" + doc: "Must be ')', a token ending the node definition." + + type_directory: + doc: "A directory node, containing a list of entries. Entries must be ordered by their names." + seq: + - id: entries + type: dir_entry + repeat: until + repeat-until: _.kind.body == ')' + types: + dir_entry: + doc: "A single entry within a directory, or a terminator." + seq: + - id: kind + type: padded_str + valid: + expr: _.body == 'entry' or _.body == ')' + doc: "Must be 'entry' (for a child node) or '' (for terminator)." + - id: open_paren + type: padded_str + valid: + expr: _.body == '(' + if: 'kind.body == "entry"' + - id: name_key + type: padded_str + valid: + expr: _.body == 'name' + if: 'kind.body == "entry"' + - id: name + type: padded_str + if: 'kind.body == "entry"' + - id: node_key + type: padded_str + valid: + expr: _.body == 'node' + if: 'kind.body == "entry"' + - id: node + type: node + if: 'kind.body == "entry"' + doc: "The child node, present only if kind is 'entry'." + - id: close_paren + type: padded_str + valid: + expr: _.body == ')' + if: 'kind.body == "entry"' + instances: + is_terminator: + value: kind.body == ')' + + type_regular: + doc: "A regular file node." + seq: + # Read attributes (like 'executable') until we hit 'contents' + - id: attributes + type: reg_attribute + repeat: until + repeat-until: _.key.body == "contents" + # After the 'contents' token, read the file data + - id: file_data + type: file_content + instances: + is_executable: + value: 'attributes[0].key.body == "executable"' + doc: "True if the file has the 'executable' attribute." + types: + reg_attribute: + doc: "An attribute of the file, e.g., 'executable' or 'contents'." + seq: + - id: key + type: padded_str + doc: "Attribute key, e.g., 'executable' or 'contents'." + valid: + expr: _.body == 'executable' or _.body == 'contents' + - id: value + type: padded_str + if: 'key.body == "executable"' + valid: + expr: _.body == '' + doc: "Must be '' if key is 'executable'." + file_content: + doc: "The raw data of the file, prefixed by length." + seq: + - id: len_contents + type: u8 + # # This relies on the property of instances that they are lazily evaluated and cached. + - size: 0 + if: nar_offset < 0 + - id: contents + size: len_contents + - id: padding + size: (8 - (len_contents % 8)) % 8 + instances: + nar_offset: + value: _io.pos + + type_symlink: + doc: "A symbolic link node." + seq: + - id: target_key + type: padded_str + doc: "Must be 'target'." + valid: + expr: _.body == 'target' + - id: target_val + type: padded_str + doc: "The destination path of the symlink." diff --git a/doc/manual/source/store/file-system-object/content-address.md b/doc/manual/source/store/file-system-object/content-address.md index 04a1021f1..5685de03e 100644 --- a/doc/manual/source/store/file-system-object/content-address.md +++ b/doc/manual/source/store/file-system-object/content-address.md @@ -46,7 +46,7 @@ be many different serialisations. For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, which is carefully designed to avoid the problems described above. -The exact specification of the Nix Archive format is in [specified here](../../protocols/nix-archive.md). +The exact specification of the Nix Archive format is in [specified here](../../protocols/nix-archive/index.md). ## Content addressing File System Objects beyond a single serialisation pass diff --git a/flake.nix b/flake.nix index e25722d46..a70617b74 100644 --- a/flake.nix +++ b/flake.nix @@ -417,6 +417,10 @@ supportsCross = false; }; + "nix-kaitai-struct-checks" = { + supportsCross = false; + }; + "nix-perl-bindings" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index f3158ea6d..c493dfad6 100644 --- a/meson.build +++ b/meson.build @@ -61,3 +61,4 @@ if get_option('unit-tests') endif subproject('nix-functional-tests') subproject('json-schema-checks') +subproject('kaitai-struct-checks') diff --git a/packaging/components.nix b/packaging/components.nix index f9d7b109a..bbd6208b9 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -443,6 +443,11 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; + /** + Kaitai struct schema validation checks + */ + nix-kaitai-struct-checks = callPackage ../src/kaitai-struct-checks/package.nix { }; + nix-perl-bindings = callPackage ../src/perl/package.nix { }; /** diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 153e7a3eb..ea12e079f 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -109,6 +109,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-kaitai-struct-checks.externalNativeBuildInputs ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 @@ -148,6 +149,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs ++ pkgs.nixComponents2.nix-cmd.buildInputs ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs - ++ lib.optional havePerl pkgs.perl; + ++ lib.optional havePerl pkgs.perl + ++ pkgs.nixComponents2.nix-kaitai-struct-checks.externalBuildInputs; } ) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 3bbb6c15b..67e2c0dfd 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -63,6 +63,7 @@ let "nix-cli" "nix-functional-tests" "nix-json-schema-checks" + "nix-kaitai-struct-checks" ] ++ lib.optionals enableBindings [ "nix-perl-bindings" diff --git a/src/kaitai-struct-checks/.version b/src/kaitai-struct-checks/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/kaitai-struct-checks/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/kaitai-struct-checks/meson.build b/src/kaitai-struct-checks/meson.build new file mode 100644 index 000000000..f705a6744 --- /dev/null +++ b/src/kaitai-struct-checks/meson.build @@ -0,0 +1,77 @@ +# Run with: +# meson test --suite kaitai-struct +# Run with: (without shell / configure) +# nix build .#nix-kaitai-struct-checks + +project( + 'nix-kaitai-struct-checks', + 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++23', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true) +gtest_dep = dependency('gtest') +gtest_main_dep = dependency('gtest_main', required : true) + +# Find the Kaitai Struct compiler +ksc = find_program('ksc', required : true) + +kaitai_generated_srcs = custom_target( + 'kaitai-generated-sources', + input : [ 'nar.ksy' ], + output : [ 'nix_nar.cpp', 'nix_nar.h' ], + command : [ + ksc, + '@INPUT@', + '--target', 'cpp_stl', + '--outdir', + meson.current_build_dir(), + ], +) + +nar_kaitai_lib = library( + 'nix-nar-kaitai-lib', + kaitai_generated_srcs, + dependencies : [ kaitai_runtime_dep ], + install : true, +) + +nar_kaitai_dep = declare_dependency( + link_with : nar_kaitai_lib, + sources : kaitai_generated_srcs[1], +) + +# The nar directory is a committed symlink to the actual nars location +nars_dir = meson.current_source_dir() / 'nars' + +# Get all example files +nars = [ + 'dot.nar', +] + +test_deps = [ + nar_kaitai_dep, + kaitai_runtime_dep, + gtest_main_dep, +] + +this_exe = executable( + meson.project_name(), + 'test-parse-nar.cc', + dependencies : test_deps, +) + +test( + meson.project_name(), + this_exe, + env : [ 'NIX_NARS_DIR=' + nars_dir ], + protocol : 'gtest', +) diff --git a/src/kaitai-struct-checks/nar.ksy b/src/kaitai-struct-checks/nar.ksy new file mode 120000 index 000000000..c3a79a3b6 --- /dev/null +++ b/src/kaitai-struct-checks/nar.ksy @@ -0,0 +1 @@ +../../doc/manual/source/protocols/nix-archive/nar.ksy \ No newline at end of file diff --git a/src/kaitai-struct-checks/nars b/src/kaitai-struct-checks/nars new file mode 120000 index 000000000..ed0b4ecc7 --- /dev/null +++ b/src/kaitai-struct-checks/nars @@ -0,0 +1 @@ +../libutil-tests/data/nars \ No newline at end of file diff --git a/src/kaitai-struct-checks/nix-meson-build-support b/src/kaitai-struct-checks/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/kaitai-struct-checks/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/kaitai-struct-checks/package.nix b/src/kaitai-struct-checks/package.nix new file mode 100644 index 000000000..263dd6fd1 --- /dev/null +++ b/src/kaitai-struct-checks/package.nix @@ -0,0 +1,75 @@ +# Run with: nix build .#nix-kaitai-struct-checks +{ + lib, + mkMesonDerivation, + gtest, + meson, + ninja, + pkg-config, + kaitai-struct-compiler, + fetchzip, + kaitai-struct-cpp-stl-runtime, + # Configuration Options + version, +}: +let + inherit (lib) fileset; +in +mkMesonDerivation (finalAttrs: { + pname = "nix-kaitai-struct-checks"; + inherit version; + + workDir = ./.; + fileset = lib.fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ./.version + ../../.version + ../../doc/manual/source/protocols/nix-archive/nar.ksy + ./nars + ../../src/libutil-tests/data + ./meson.build + ./nar.ksy + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" ]; + + passthru.externalNativeBuildInputs = [ + # This can go away when we bump up to 25.11 + (kaitai-struct-compiler.overrideAttrs (finalAttrs: { + version = "0.11"; + src = fetchzip { + url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip"; + sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A="; + }; + })) + ]; + + passthru.externalBuildInputs = [ + gtest + kaitai-struct-cpp-stl-runtime + ]; + + buildInputs = finalAttrs.passthru.externalBuildInputs; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ] + ++ finalAttrs.passthru.externalNativeBuildInputs; + + doCheck = true; + + mesonCheckFlags = [ "--print-errorlogs" ]; + + postInstall = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/kaitai-struct-checks/test-parse-nar.cc b/src/kaitai-struct-checks/test-parse-nar.cc new file mode 100644 index 000000000..456ffb127 --- /dev/null +++ b/src/kaitai-struct-checks/test-parse-nar.cc @@ -0,0 +1,48 @@ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "nix_nar.h" + +static const std::vector NarFiles = { + "empty.nar", + "dot.nar", + "dotdot.nar", + "executable-after-contents.nar", + "invalid-tag-instead-of-contents.nar", + "name-after-node.nar", + "nul-character.nar", + "slash.nar", +}; + +class NarParseTest : public ::testing::TestWithParam +{}; + +TEST_P(NarParseTest, ParseSucceeds) +{ + const auto nar_file = GetParam(); + + const char * nars_dir_env = std::getenv("NIX_NARS_DIR"); + if (nars_dir_env == nullptr) { + FAIL() << "NIX_NARS_DIR environment variable not set."; + } + + const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar"; + ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path; + + std::ifstream ifs(nar_file_path, std::ifstream::binary); + ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file; + kaitai::kstream ks(&ifs); + nix_nar_t nar(&ks); + ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file; +} + +INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles)); diff --git a/src/nix/nar.md b/src/nix/nar.md index b0f70ce93..c29c2092a 100644 --- a/src/nix/nar.md +++ b/src/nix/nar.md @@ -8,7 +8,7 @@ R""( # File format For the definition of the Nix Archive file format, see -[within the protocols chapter](@docroot@/protocols/nix-archive.md) +[within the protocols chapter](@docroot@/protocols/nix-archive/index.md) of the manual. [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive From 72d0f7b61941225eb06762095131f2b42cd3a56a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 Oct 2025 02:16:50 -0400 Subject: [PATCH 119/155] Document "hash derivation quotiented", resolution, and build trace Progress on #13405, which asks for an explicit characterisation of the equivalence relation like the one given here. Also progress on #11895, because we're using the term "build trace entry" instead of "realisation". Mention #9259, a future work item. Co-authored-by: Robert Hensing --- doc/manual/book.toml.in | 1 + doc/manual/meson.build | 2 + doc/manual/source/SUMMARY.md.in | 3 + .../source/protocols/derivation-aterm.md | 4 +- .../protocols/json/schema/derivation-v3.yaml | 4 +- doc/manual/source/store/build-trace.md | 53 +++++ doc/manual/source/store/derivation/index.md | 2 +- .../derivation/outputs/content-address.md | 4 +- .../store/derivation/outputs/input-address.md | 225 ++++++++++++++++-- doc/manual/source/store/math-notation.md | 16 ++ doc/manual/source/store/resolution.md | 219 +++++++++++++++++ doc/manual/theme/head.hbs | 15 ++ src/libstore/include/nix/store/derivations.hh | 2 +- 13 files changed, 528 insertions(+), 22 deletions(-) create mode 100644 doc/manual/source/store/build-trace.md create mode 100644 doc/manual/source/store/math-notation.md create mode 100644 doc/manual/source/store/resolution.md create mode 100644 doc/manual/theme/head.hbs diff --git a/doc/manual/book.toml.in b/doc/manual/book.toml.in index 34acf642e..bacca59ff 100644 --- a/doc/manual/book.toml.in +++ b/doc/manual/book.toml.in @@ -7,6 +7,7 @@ additional-css = ["custom.css"] additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +mathjax-support = true # Handles replacing @docroot@ with a path to ./source relative to that markdown file, # {{#include handlebars}}, and the @generated@ syntax used within these. it mostly diff --git a/doc/manual/meson.build b/doc/manual/meson.build index fdea40098..231f7b9f8 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -92,6 +92,8 @@ manual = custom_target( (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 rm -rf @2@/manual mv @2@/html @2@/manual + # Remove Mathjax 2.7, because we will actually use MathJax 3.x + find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d' find @2@/manual -iname meson.build -delete '''.format( python.full_path(), diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 287dff872..b87bf93a3 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -26,9 +26,12 @@ - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) - [Input-addressing derivation outputs](store/derivation/outputs/input-address.md) + - [Build Trace](store/build-trace.md) + - [Derivation Resolution](store/resolution.md) - [Building](store/building.md) - [Store Types](store/types/index.md) {{#include ./store/types/SUMMARY.md}} + - [Appendix: Math notation](store/math-notation.md) - [Nix Language](language/index.md) - [Data Types](language/types.md) - [String context](language/string-context.md) diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 99e3c2be6..523678e66 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -1,6 +1,8 @@ # Derivation "ATerm" file format -For historical reasons, [store derivations][store derivation] are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. +For historical reasons, [store derivations][store derivation] are stored on-disk in "Annotated Term" (ATerm) format +([guide](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html), +[paper](https://doi.org/10.1002/(SICI)1097-024X(200003)30:3%3C259::AID-SPE298%3E3.0.CO;2-Y)). ## The ATerm format used diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml index 9c0210bb7..fa68adcb1 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v3.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -39,9 +39,9 @@ properties: This is a guard that allows us to continue evolving this format. The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - Version 0: A-Term format + - Version 0: ATerm format - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - Version 2: Separate `method` and `hashAlgo` fields in output specs diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md new file mode 100644 index 000000000..1086dcb88 --- /dev/null +++ b/doc/manual/source/store/build-trace.md @@ -0,0 +1,53 @@ +# Build Trace + +> **Warning** +> +> This entire concept is currently +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) +> and subject to change. + +The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. +It maps the inputs of builds to the outputs of builds. +Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. + +In general the derivations used as a key should be [*resolved*](./resolution.md). +A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. +If all the resolved inputs of a derivation are content-addressed, that means the inputs will be fully determined, leaving no ambiguity for what build was performed. +(Input-addressed inputs however are still ambiguous. They too should be locked down, but this is left as future work.) + +Accordingly, to look up an unresolved derivation, one must first resolve it to get a resolved derivation. +Resolving itself involves looking up entries in the build trace, so this is a mutually recursive process that will end up inspecting possibly many entries. + +Except for the issue with input-addressed paths called out above, base build traces are trivially *coherent* -- incoherence is not possible. +That means that the claims that each key-value base build try entry makes are independent, and no mapping invalidates another mapping. + +Whether the mappings are *true*, i.e. the faithful recording of actual builds performed, is another matter. +Coherence is about the multiple claims of the build trace being mutually consistent, not about whether the claims are individually true or false. + +In general, there is no way to audit a build trace entry except for by performing the build again from scratch. +And even in that case, a different result doesn't mean the original entry was a "lie", because the derivation being built may be non-deterministic. +As such, the decision of whether to trust a counterparty's build trace is a fundamentally subject policy choice. +Build trace entries are typically *signed* in order to enable arbitrary public-key-based trust polices. + +## Derived build traces + +Implementations that wish to memoize the above may also keep additional *derived* build trace entries that do map unresolved derivations. +But if they do so, they *must* also keep the underlying base entries with resolved derivation keys around. +Firstly, this ensures that the derived entries are merely cache, which could be recomputed from scratch. +Secondly, this ensures the coherence of the derived build trace. + +Unlike with base build traces, incoherence with derived build traces is possible. +The key ingredient is that derivation resolution is only deterministic with respect to a fixed base build trace. +Without fixing the base build trace, it inherits the subjectivity of base build traces themselves. + +Concretely, suppose there are three derivations \\(a\\), \\(b\\), and \((c\\). +Let \\(a\\) be a resolved derivation, but let \\(b\\) and \((c\\) be unresolved and both take as an input an output of \\(a\\). +Now suppose that derived entries are made for \\(b\\) and \((c\\) based on two different entries of \\(a\\). +(This could happen if \\(a\\) is non-deterministic, \\(a\\) and \\(b\\) are built in one store, \\(a\\) and \\(c\\) are built in another store, and then a third store substitutes from both of the first two stores.) + +If trusting the derived build trace entries for \\(b\\) and \((c\\) requires that each's underlying entry for \\(a\\) be also trusted, the two different mappings for \\(a\\) will be caught. +However, if \\(b\\) and \((c\\)'s entries can be combined in isolation, there will be nothing to catch the contradiction in their hidden assumptions about \\(a\\)'s output. + +[derivation]: ./derivation/index.md +[output]: ./derivation/outputs/index.md +[store object]: @docroot@/store/store-object.md diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 5b179273d..61c5335ff 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -245,7 +245,7 @@ If those other derivations *also* abide by this common case (and likewise for tr > note the ".drv" > ``` -## Extending the model to be higher-order +## Extending the model to be higher-order {#dynamic} **Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index 4d5130348..aa65fbe49 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -167,10 +167,10 @@ It is only in the potential for that check to fail that they are different. > > In a future world where floating content-addressing is also stable, we in principle no longer need separate [fixed](#fixed) content-addressing. > Instead, we could always use floating content-addressing, and separately assert the precise value content address of a given store object to be used as an input (of another derivation). -> A stand-alone assertion object of this sort is not yet implemented, but its possible creation is tracked in [Issue #11955](https://github.com/NixOS/nix/issues/11955). +> A stand-alone assertion object of this sort is not yet implemented, but its possible creation is tracked in [issue #11955](https://github.com/NixOS/nix/issues/11955). > > In the current version of Nix, fixed outputs which fail their hash check are still registered as valid store objects, just not registered as outputs of the derivation which produced them. -> This is an optimization that means if the wrong output hash is specified in a derivation, and then the derivation is recreated with the right output hash, derivation does not need to be rebuilt --- avoiding downloading potentially large amounts of data twice. +> This is an optimization that means if the wrong output hash is specified in a derivation, and then the derivation is recreated with the right output hash, derivation does not need to be rebuilt — avoiding downloading potentially large amounts of data twice. > This optimisation prefigures the design above: > If the output hash assertion was removed outside the derivation itself, Nix could additionally not only register that outputted store object like today, but could also make note that derivation did in fact successfully download some data. For example, for the "fetch URL" example above, making such a note is tantamount to recording what data is available at the time of download at the given URL. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index e2e15a801..3fd20f17d 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -6,26 +6,221 @@ That is to say, an input-addressed output's store path is a function not of the output itself, but of the derivation that produced it. Even if two store paths have the same contents, if they are produced in different ways, and one is input-addressed, then they will have different store paths, and thus guaranteed to not be the same store object. - +type FirstOrderDerivingPath = ConstantPath | FirstOrderOutputPath; +type Inputs = Set; +``` + +For the algorithm below, we adopt a derivation where the two types of (first order) derived paths are partitioned into two sets, as follows: +```typescript +type Derivation = { + // inputs: Set; // replaced + inputSrcs: Set; // new instead + inputDrvOutputs: Set; // new instead + // ...other fields... +}; +``` + +In the [currently-experimental][xp-feature-dynamic-derivations] higher-order case where outputs of outputs are allowed as [deriving paths][deriving-path] and thus derivation inputs, derivations using that generalization are not valid arguments to this function. +Those derivations must be (partially) [resolved](@docroot@/store/resolution.md) enough first, to the point where no such higher-order inputs remain. +Then, and only then, can input addresses be assigned. + +``` +function hashQuotientDerivation(drv) -> Hash: + assert(drv.outputs are input-addressed) + drv′ ← drv with { + inputDrvOutputs = ⋃( + assert(drvPath is store path) + case hashOutputsOrQuotientDerivation(readDrv(drvPath)) of + drvHash : Hash → + (drvHash.toBase16(), output) + outputHashes : Map[String, Hash] → + (outputHashes[output].toBase16(), "out") + | (drvPath, output) ∈ drv.inputDrvOutputs + ) + } + return hashSHA256(printDrv(drv′)) + +function hashOutputsOrQuotientDerivation(drv) -> Map[String, Hash] | Hash: + if drv.outputs are content-addressed: + return { + outputName ↦ hashSHA256( + "fixed:out:" + ca.printMethodAlgo() + + ":" + ca.hash.toBase16() + + ":" + ca.makeFixedOutputPath(drv.name, outputName)) + | (outputName ↦ output) ∈ drv.outputs + , ca = output.contentAddress // or get from build trace if floating + } + else: // drv.outputs are input-addressed + return hashQuotientDerivation(drv) +``` + +### `hashQuotientDerivation` + +We replace each element in the derivation's `inputDrvOutputs` using data from a call to `hashOutputsOrQuotientDerivation` on the `drvPath` of that element. +When `hashOutputsOrQuotientDerivation` returns a single drv hash (because the input derivation in question is input-addressing), we simply swap out the `drvPath` for that hash, and keep the same output name. +When `hashOutputsOrQuotientDerivation` returns a map of content addresses per-output, we look up the output in question, and pair it with the output name `out`. + +The resulting pseudo-derivation (with hashes instead of store paths in `inputDrvs`) is then printed (in the ["ATerm" format](@docroot@/protocols/derivation-aterm.md)) and hashed, and this becomes the hash of the "quotient derivation". + +When calculating output hashes, `hashQuotientDerivation` is called on an almost-complete input-addressing derivation, which is just missing its input-addressed outputs paths. +The derivation hash is then used to calculate output paths for each output. + +Those output paths can then be substituted into the almost-complete input-addressed derivation to complete it. + +> **Note** +> +> There may be an unintentional deviation from specification currently implemented in the `(outputHashes[output].toBase16(), "out")` case. +> This is not fatal because the deviation would only apply for content-addressing derivations with more than one output, and that only occurs in the floating case, which is [experimental][xp-feature-ca-derivations]. +> Once this bug is fixed, this note will be removed. + +### `hashOutputsOrQuotientDerivation` + +How does `hashOutputsOrQuotientDerivation` in turn work? +It consists of two main cases, based on whether the outputs of the derivation are to be input-addressed or content-addressed. + +#### Input-addressed outputs case + +In the input-addressed case, it just calls `hashQuotientDerivation`, and returns that derivation hash. +This makes `hashQuotientDerivation` and `hashOutputsOrQuotientDerivation` mutually-recursive. + +> **Note** +> +> In this case, `hashQuotientDerivation` is being called on a *complete* input-addressing derivation that already has its output paths calculated. +> The `inputDrvs` substitution takes place anyways. + +#### Content-addressed outputs case + +If the outputs are [content-addressed](./content-address.md), then it computes a hash for each output derived from the content-address of that output. + +> **Note** +> +> In the [fixed](./content-address.md#fixed) content-addressing case, the outputs' content addresses are statically specified in advance, so this always just works. +> (The fixed case is what the pseudo-code shows.) +> +> In the [floating](./content-address.md#floating) case, the content addresses are not specified in advance. +> This is what the "or get from [build trace](@docroot@/store/build-trace.md) if floating" comment refers to. +> In this case, the algorithm is *stuck* until the input in question is built, and we know what the actual contents of the output in question is. +> +> That is OK however, because there is no problem with delaying the assigning of input addresses (which, remember, is what `hashQuotientDerivation` is ultimately for) until all inputs are known. + +### Performance + +The recursion in the algorithm is potentially inefficient: +it could call itself once for each path by which a subderivation can be reached, i.e., `O(V^k)` times for a derivation graph with `V` derivations and with out-degree of at most `k`. +In the actual implementation, [memoisation](https://en.wikipedia.org/wiki/Memoization) is used to reduce this cost to be proportional to the total number of `inputDrvOutputs` encountered. + +### Semantic properties + +*See [this chapter's appendix](@docroot@/store/math-notation.md) on grammar and metavariable conventions.* + +In essence, `hashQuotientDerivation` partitions input-addressing derivations into equivalence classes: every derivation in that equivalence class is mapped to the same derivation hash. +We can characterize this equivalence relation directly, by working bottom up. + +We start by defining an equivalence relation on first-order output deriving paths that refer content-addressed derivation outputs. Two such paths are equivalent if they refer to the same store object: + +\\[ +\\begin{prooftree} +\\AxiomC{$d\_1$ is content-addressing} +\\AxiomC{$d\_2$ is content-addressing} +\\AxiomC{$ + {}^\*(\text{path}(d\_1), o\_1) + \= + {}^\*(\text{path}(d\_2), o\_2) +$} +\\TrinaryInfC{$(\text{path}(d\_1), o\_1) \\,\\sim_{\\mathrm{CA}}\\, (d\_2, o\_2)$} +\\end{prooftree} +\\] + +where \\({}^*(s, o)\\) denotes the store object that the output deriving path refers to. + +We will also need the following construction to lift any equivalence relation on \\(X\\) to an equivalence relation on (finite) sets of \\(X\\) (in short, \\(\\mathcal{P}(X)\\)): + +\\[ +\\begin{prooftree} +\\AxiomC{$\\forall a \\in A. \\exists b \\in B. a \\,\\sim\_X\\, b$} +\\AxiomC{$\\forall b \\in B. \\exists a \\in A. b \\,\\sim\_X\\, a$} +\\BinaryInfC{$A \\,\\sim_{\\mathcal{P}(X)}\\, B$} +\\end{prooftree} +\\] + +Now we can define the equivalence relation \\(\\sim_\\mathrm{IA}\\) on input-addressed derivation outputs. Two input-addressed outputs are equivalent if their derivations are equivalent (via the yet-to-be-defined \\(\\sim_{\\mathrm{IADrv}}\\) relation) and their output names are the same: + +\\[ +\\begin{prooftree} +\\AxiomC{$d\_1$ is input-addressing} +\\AxiomC{$d\_2$ is input-addressing} +\\AxiomC{$d\_1 \\,\\sim_{\\mathrm{IADrv}}\\, d\_2$} +\\AxiomC{$o\_1 = o\_2$} +\\QuaternaryInfC{$(\text{path}(d\_1), o\_1) \\,\\sim_{\\mathrm{IA}}\\, (\text{path}(d\_2), o\_2)$} +\\end{prooftree} +\\] + +And now we can define \\(\\sim_{\\mathrm{IADrv}}\\). +Two input-addressed derivations are equivalent if their content-addressed inputs are equivalent, their input-addressed inputs are also equivalent, and they are otherwise equal: + + + +\\[ +\\begin{prooftree} +\\alwaysNoLine +\\AxiomC{$ + \\mathrm{caInputs}(d\_1) + \\,\\sim_{\\mathcal{P}(\\mathrm{CA})}\\, + \\mathrm{caInputs}(d\_2) +$} +\\AxiomC{$ + \\mathrm{iaInputs}(d\_1) + \\,\\sim_{\\mathcal{P}(\\mathrm{IA})}\\, + \\mathrm{iaInputs}(d\_2) +$} +\\BinaryInfC{$ + d\_1\left[\\mathrm{inputDrvOutputs} := \\{\\}\right] + \= + d\_2\left[\\mathrm{inputDrvOutputs} := \\{\\}\right] +$} +\\alwaysSingleLine +\\UnaryInfC{$d\_1 \\,\\sim_{\\mathrm{IADrv}}\\, d\_2$} +\\end{prooftree} +\\] + +where \\(\\mathrm{caInputs}(d)\\) returns the content-addressed inputs of \\(d\\) and \\(\\mathrm{iaInputs}(d)\\) returns the input-addressed inputs. + +> **Note** +> +> An astute reader might notice that that nowhere does `inputSrcs` enter into these definitions. +> That means that replacing an input derivation with its outputs directly added to `inputSrcs` always results in a derivation in a different equivalence class, despite the resulting input closure (as would be mounted in the store at build time) being the same. +> [Issue #9259](https://github.com/NixOS/nix/issues/9259) is about creating a coarser equivalence relation to address this. +> +> \\(\\sim_\mathrm{Drv}\\) from [derivation resolution](@docroot@/store/resolution.md) is such an equivalence relation. +> It is coarser than this one: any two derivations which are "'hash quotient derivation'-equivalent" (\\(\\sim_\mathrm{IADrv}\\)) are also "resolution-equivalent" (\\(\\sim_\mathrm{Drv}\\)). +> It also relates derivations whose `inputDrvOutputs` have been rewritten into `inputSrcs`. + +[deriving-path]: @docroot@/store/derivation/index.md#deriving-path +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations [xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations -[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing -[xp-feature-impure-derivations]: @docroot@/development/experimental-features.md#xp-feature-impure-derivations diff --git a/doc/manual/source/store/math-notation.md b/doc/manual/source/store/math-notation.md new file mode 100644 index 000000000..723982e73 --- /dev/null +++ b/doc/manual/source/store/math-notation.md @@ -0,0 +1,16 @@ +# Appendix: Math notation + +A few times in this manual, formal "proof trees" are used for [natural deduction](https://en.wikipedia.org/wiki/Natural_deduction)-style definition of various [relations](https://en.wikipedia.org/wiki/Relation_(mathematics)). + +The following grammar and assignment of metavariables to syntactic categories is used in these sections. + +\\begin{align} +s, t &\in \text{store-path} \\\\ +o &\in \text{output-name} \\\\ +i, p &\in \text{deriving-path} \\\\ +d &\in \text{derivation} +\\end{align} + +\\begin{align} +\text{deriving-path} \quad p &::= s \mid (p, o) +\\end{align} diff --git a/doc/manual/source/store/resolution.md b/doc/manual/source/store/resolution.md new file mode 100644 index 000000000..9a87fea99 --- /dev/null +++ b/doc/manual/source/store/resolution.md @@ -0,0 +1,219 @@ +# Derivation Resolution + +*See [this chapter's appendix](@docroot@/store/math-notation.md) on grammar and metavariable conventions.* + +To *resolve* a derivation is to replace its [inputs] with the simplest inputs — plain store paths — that denote the same store objects. + +Derivations that only have store paths as inputs are likewise called *resolved derivations*. +(They are called that whether they are in fact the output of derivation resolution, or just made that way without non-store-path inputs to begin with.) + +## Input Content Equivalence of Derivations + +[Deriving paths][deriving-path] intentionally make it possible to refer to the same [store object] in multiple ways. +This is a consequence of content-addressing, since different derivations can produce the same outputs, and the same data can also be manually added to the store. +This is also a consequence even of input-addressing, as an output can be referred to by derivation and output name, or directly by its [computed](./derivation/outputs/input-address.md) store path. +Since dereferencing deriving paths is thus not injective, it induces an equivalence relation on deriving paths. + +Let's call this equivalence relation \\(\\sim\\), where \\(p_1 \\sim p_2\\) means that deriving paths \\(p_1\\) and \\(p_2\\) refer to the same store object. + +**Content Equivalence**: Two deriving paths are equivalent if they refer to the same store object: + +\\[ +\\begin{prooftree} +\\AxiomC{${}^*p_1 = {}^*p_2$} +\\UnaryInfC{$p_1 \\,\\sim_\\mathrm{DP}\\, p_2$} +\\end{prooftree} +\\] + +where \\({}^\*p\\) denotes the store object that deriving path \\(p\\) refers to. + +This also induces an equivalence relation on sets of deriving paths: + +\\[ +\\begin{prooftree} +\\AxiomC{$\\{ {}^*p | p \\in P_1 \\} = \\{ {}^*p | p \\in P_2 \\}$} +\\UnaryInfC{$P_1 \\,\\sim_{\\mathcal{P}(\\mathrm{DP})}\\, P_2$} +\\end{prooftree} +\\] + +**Input Content Equivalence**: This, in turn, induces an equivalence relation on derivations: two derivations are equivalent if their inputs are equivalent, and they are otherwise equal: + +\\[ +\\begin{prooftree} +\\AxiomC{$\\mathrm{inputs}(d_1) \\,\\sim_{\\mathcal{P}(\\mathrm{DP})}\\, \\mathrm{inputs}(d_2)$} +\\AxiomC{$ + d\_1\left[\\mathrm{inputs} := \\{\\}\right] + \= + d\_2\left[\\mathrm{inputs} := \\{\\}\right] +$} +\\BinaryInfC{$d_1 \\,\\sim_\\mathrm{Drv}\\, d_2$} +\\end{prooftree} +\\] + +Derivation resolution always maps derivations to input-content-equivalent derivations. + +## Resolution relation + +Dereferencing a derived path — \\({}^\*p\\) above — was just introduced as a black box. +But actually it is a multi-step process of looking up build results in the [build trace] that itself depends on resolving the lookup keys. +Resolution is thus a recursive multi-step process that is worth diagramming formally. + +We can do this with a small-step binary transition relation; let's call it \\(\rightsquigarrow\\). +We can then conclude dereferenced equality like this: + +\\[ +\\begin{prooftree} +\\AxiomC{$p\_1 \\rightsquigarrow^* p$} +\\AxiomC{$p\_2 \\rightsquigarrow^* p$} +\\BinaryInfC{${}^*p\_1 = {}^*p\_2$} +\\end{prooftree} +\\] + +I.e. by showing that both original items resolve (over 0 or more small steps, hence the \\({}^*\\)) to the same exact item. + +With this motivation, let's now formalize a [small-step](https://en.wikipedia.org/wiki/Operational_semantics#Small-step_semantics) system of reduction rules for resolution. + +### Formal rules + +### \\(\text{resolved}\\) unary relation + +\\[ +\\begin{prooftree} +\\AxiomC{$s \in \text{store-path}$} +\\UnaryInfC{$s$ resolved} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$\forall i \in \mathrm{inputs}(d). i \text{ resolved}$} +\\UnaryInfC{$d$ resolved} +\\end{prooftree} +\\] + +### \\(\rightsquigarrow\\) binary relation + +> **Remark** +> +> Actually, to be completely formal we would need to keep track of the build trace we are choosing to resolve against. +> +> We could do that by making \\(\rightsquigarrow\\) a ternary relation, which would pass the build trace to itself until it finally uses it in that one rule. +> This would add clutter more than insight, so we didn't bother to write it. +> +> There are other options too, like saying the whole reduction rule system is parameterized on the build trace, essentially [currying](https://en.wikipedia.org/wiki/Currying) the ternary \\(\rightsquigarrow\\) into a function from build traces to the binary relation written above. + +#### Core build trace lookup rule + +\\[ +\\begin{prooftree} +\\AxiomC{$s \in \text{store-path}$} +\\AxiomC{${}^*s \in \text{derivation}$} +\\AxiomC{${}^*s$ resolved} +\\AxiomC{$\mathrm{build\text{-}trace}[s][o] = t$} +\\QuaternaryInfC{$(s, o) \rightsquigarrow t$} +\\RightLabel{\\scriptsize output path resolution} +\\end{prooftree} +\\] + +#### Inductive rules + +\\[ +\\begin{prooftree} +\\AxiomC{$i \\rightsquigarrow i'$} +\\AxiomC{$i \\in \\mathrm{inputs}(d)$} +\\BinaryInfC{$d \\rightsquigarrow d[i \\mapsto i']$} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$d \\rightsquigarrow d'$} +\\UnaryInfC{$(\\mathrm{path}(d), o) \\rightsquigarrow (\\mathrm{path}(d'), o)$} +\\end{prooftree} +\\] + +\\[ +\\begin{prooftree} +\\AxiomC{$p \\rightsquigarrow p'$} +\\UnaryInfC{$(p, o) \\rightsquigarrow (p', o)$} +\\end{prooftree} +\\] + +### Properties + +Like all well-behaved evaluation relations, partial resolution is [*confluent*](https://en.wikipedia.org/wiki/Confluence_(abstract_rewriting)). +Also, if we take the symmetric closure of \\(\\rightsquigarrow^\*\\), we end up with the equivalence relations of the previous section. +Resolution respects content equivalence for deriving paths, and input content equivalence for derivations. + +> **Remark** +> +> We chose to define from scratch an "resolved" unary relation explicitly above. +> But it can also be defined as the normal forms of the \\(\\rightsquigarrow^\*\\) relation: +> +> \\[ a \text{ resolved} \Leftrightarrow \forall b. b \rightsquigarrow^* a \Rightarrow b = a\\] +> +> In prose, resolved terms are terms which \\(\\rightsquigarrow^\*\\) only relates on the left side to the same term on the right side; they are the terms which can be resolved no further. + +## Partial versus Complete Resolution + +Similar to evaluation, we can also speak of *partial* versus *complete* derivation resolution. +Partial derivation resolution is what we've actually formalized above with \\(\\rightsquigarrow^\*\\). +Complete resolution is resolution ending in a resolved term (deriving path or derivation). +(Which is a normal form of the relation, per the remark above.) + +With partial resolution, a derivation is related to equivalent derivations with the same or simpler inputs, but not all those inputs will be plain store paths. +This is useful when the input refers to a floating content addressed output we have not yet built — we don't know what (content-address) store path will used for that derivation, so we are "stuck" trying to resolve the deriving path in question. +(In the above formalization, this happens when the build trace is missing the keys we wish to look up in it.) + +Complete resolution is a *functional* relation, i.e. values on the left are uniquely related with values on the right. +It is not however, a *total* relation (in general, assuming arbitrary build traces). +This is discussed in the next section. + +## Termination + +For static derivations graphs, complete resolution is indeed total, because it always terminates for all inputs. +(A relation that is both total and functional is a function.) + +For [dynamic][xp-feature-dynamic-derivations] derivation graphs, however, this is not the case — resolution is not guaranteed to terminate. +The issue isn't rewriting deriving paths themselves: +a single rewrite to normalize an output deriving path to a constant one always exists, and always proceeds in one step. +The issue is that dynamic derivations (i.e. those that are filled-in the graph by a previous resolution) may have more transitive dependencies than the original derivation. + +> **Example** +> +> Suppose we have this deriving path +> ```json +> { +> "drvPath": { +> "drvPath": "...-foo.drv", +> "output": "bar.drv" +> }, +> "output": "baz" +> } +> ``` +> and derivation `foo` is already resolved. +> When we resolve deriving path we'll end up with something like. +> ```json +> { +> "drvPath": "...-foo-bar.drv", +> "output": "baz" +> } +> ``` +> So far is just an atomic single rewrite, with no termination issues. +> But the derivation `foo-bar` may have its *own* dynamic derivation inputs. +> Resolution must resolve that derivation first before the above deriving path can finally be normalized to a plain `...-foo-bar-baz` store path. + +The important thing to notice is that while "build trace" *keys* must be resolved. +The *value* those keys are mapped to have no such constraints. +An arbitrary store object has no notion of being resolved or not. +But, an arbitrary store object can be read back as a derivation (as will in fact be done in case for dynamic derivations / nested output deriving paths). +And those derivations need *not* be resolved. + +It is those dynamic non-resolved derivations which are the source of non-termination. +By the same token, they are also the reason why dynamic derivations offer greater expressive power. + +[store object]: @docroot@/store/store-object.md +[inputs]: @docroot@/store/derivation/index.md#inputs +[build trace]: @docroot@/store/build-trace.md +[deriving-path]: @docroot@/store/derivation/index.md#deriving-path +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs new file mode 100644 index 000000000..e514a9977 --- /dev/null +++ b/doc/manual/theme/head.hbs @@ -0,0 +1,15 @@ + + + diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 4615d8acd..259314d3f 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -277,7 +277,7 @@ struct BasicDerivation Path builder; Strings args; /** - * Must not contain the key `__json`, at least in order to serialize to A-Term. + * Must not contain the key `__json`, at least in order to serialize to ATerm. */ StringPairs env; std::optional structuredAttrs; From 0d7b16da4dae9dc8f07756a6ef87caf17406e835 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Oct 2025 13:03:18 -0400 Subject: [PATCH 120/155] Split realisation protocol unit tests This will allow us to more accurately test dropping support for dependent realisations, by separating the tests that should not change from the tests that should. I do that change in PR #14247, but even if for some reasons we don't end up doing this soon, I think it is still good to separate the test data this way so we have the option of doing that at some point. --- src/libstore-tests/common-protocol.cc | 19 +++++++++++++-- .../common-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/common-protocol/realisation.bin | Bin 520 -> 384 bytes .../serve-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/serve-protocol/realisation.bin | Bin 520 -> 384 bytes .../worker-protocol/realisation-with-deps.bin | Bin 0 -> 320 bytes .../data/worker-protocol/realisation.bin | Bin 520 -> 384 bytes src/libstore-tests/serve-protocol.cc | 17 +++++++++++++ src/libstore-tests/worker-protocol.cc | 23 +++++++++++++++--- 9 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 src/libstore-tests/data/common-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-with-deps.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-with-deps.bin diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 2c001957b..7c40e8cdb 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -114,13 +114,28 @@ CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +CHARACTERIZATION_TEST( + realisation_with_deps, + "realisation-with-deps", + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin new file mode 100644 index 0000000000000000000000000000000000000000..54a78b64ebcf0726bd7da506434a316c7336e6e9 GIT binary patch literal 320 zcmXqFWB`L|rIgfy)V!3`ypo{Q#GK6H#FEVXykaG*YNg_gL?cr(E3-5UGs`r~)I=i- zBjco$L_;&vR0A^ubF;J*gOpSgV>7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg7dq)Wj5{WP?bMq9FG(*#5V{_v) zQ^ms4(h4OjrF6q`^NdR4LR_?NT7JG#t&UP=ijoz~ZbQ>l<787a0}D%&X3lj^AWJ^m+6HD_n6H{ZuBr{6`^F(t~3&Z48vlLS!bH${@DkUAI v{L+$u#F7kR9igLCoSB}NSW;S)TC8Lht&~`tlBT4iR9K!`q!e2V4mJh=YNB5O literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 2176c6c4afd96b32fe372de6084ac7f4c7a11d49..3a0b2b2d8e393e8786bec9d1f3dec9fb1d17ce13 100644 GIT binary patch delta 14 VcmeBRX<%mDFp+U9Q*7;o696H%1v~%% delta 107 zcmZo*?qFfuJCSkg{ + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -104,6 +113,14 @@ VERSIONED_CHARACTERIZATION_TEST( .outputName = "baz", }, }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index c4afde3bd..8f70e937b 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -150,13 +150,30 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, }, + Realisation{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation_with_deps, + "realisation-with-deps", + defaultVersion, + (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -172,7 +189,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, }, }, - DrvOutput{ + { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), .outputName = "baz", }, From 144c66215b49bdadba7e11d2fd3ff9b108750274 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Oct 2025 18:01:37 -0400 Subject: [PATCH 121/155] JSON Schema for build trace entry Note, starting to make progress on #11895 by calling it this in the manual. --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/build-trace-entry.md | 27 +++++++ doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/build-trace-entry-v1 | 1 + .../json/schema/build-trace-entry-v1.yaml | 74 +++++++++++++++++++ doc/manual/source/store/build-trace.md | 2 +- src/json-schema-checks/build-trace-entry | 1 + src/json-schema-checks/meson.build | 9 +++ src/json-schema-checks/package.nix | 1 + 10 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/protocols/json/build-trace-entry.md create mode 120000 doc/manual/source/protocols/json/schema/build-trace-entry-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml create mode 120000 src/json-schema-checks/build-trace-entry diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 7d29df3c3..e13c6f33d 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -37,6 +37,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index b87bf93a3..580076ece 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -126,6 +126,7 @@ - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) + - [Build Trace Entry](protocols/json/build-trace-entry.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md new file mode 100644 index 000000000..8050a2840 --- /dev/null +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -0,0 +1,27 @@ +{{#include build-trace-entry-v1-fixed.md}} + +## Examples + +### Simple build trace entry + +```json +{{#include schema/build-trace-entry-v1/simple.json}} +``` + +### Build trace entry with dependencies + +```json +{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}} +``` + +### Build trace entry with signature + +```json +{{#include schema/build-trace-entry-v1/with-signature.json}} +``` + + \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 7ebcff697..d8e94d68c 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -15,6 +15,7 @@ schemas = [ 'store-object-info-v1', 'derivation-v3', 'deriving-path-v1', + 'build-trace-entry-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1 b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 new file mode 120000 index 000000000..0d02880a5 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml new file mode 100644 index 000000000..cabf2c350 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml @@ -0,0 +1,74 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json" +title: Build Trace Entry +description: | + A record of a successful build outcome for a specific derivation output. + + This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +type: object +required: + - id + - outPath + - dependentRealisations + - signatures +properties: + id: + type: string + title: Derivation Output ID + pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + description: | + Unique identifier for the derivation output that was built. + + Format: `{hash-quotient-drv}!{output-name}` + + - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). + Begins with `sha256:`. + + - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") + + Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + + outPath: + "$ref": "store-path-v1.yaml" + title: Output Store Path + description: | + The path to the store object that resulted from building this derivation for the given output name. + + dependentRealisations: + type: object + title: Underlying Base Build Trace + description: | + This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. + + Keys are derivation output IDs (same format as the main `id` field). + Values are the store paths that those dependencies resolved to. + + As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. + This is the set of base build trace entries that this derived build trace is derived from. + (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) + + patternProperties: + "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": + $ref: "store-path-v1.yaml" + title: Dependent Store Path + description: Store path that this dependency resolved to during the build + additionalProperties: false + + signatures: + type: array + title: Build Signatures + description: | + A set of cryptographic signatures attesting to the authenticity of this build trace entry. + items: + type: string + title: Signature + description: A single cryptographic signature + +additionalProperties: false diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index 1086dcb88..8860bc6c7 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -29,7 +29,7 @@ And even in that case, a different result doesn't mean the original entry was a As such, the decision of whether to trust a counterparty's build trace is a fundamentally subject policy choice. Build trace entries are typically *signed* in order to enable arbitrary public-key-based trust polices. -## Derived build traces +## Derived build traces {#derived} Implementations that wish to memoize the above may also keep additional *derived* build trace entries that do map unresolved derivations. But if they do so, they *must* also keep the underlying base entries with resolved derivation keys around. diff --git a/src/json-schema-checks/build-trace-entry b/src/json-schema-checks/build-trace-entry new file mode 120000 index 000000000..9175e750e --- /dev/null +++ b/src/json-schema-checks/build-trace-entry @@ -0,0 +1 @@ +../../src/libstore-tests/data/realisation \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 67f553162..c2c7fbff4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -54,6 +54,15 @@ schemas = [ 'single_built_built.json', ], }, + { + 'stem' : 'build-trace-entry', + 'schema' : schema_dir / 'build-trace-entry-v1.yaml', + 'files' : [ + 'simple.json', + 'with-dependent-realisations.json', + 'with-signature.json', + ], + }, ] # Derivation and Derivation output diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 160db003f..057a6e85b 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -23,6 +23,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path + ../../src/libstore-tests/data/realisation ../../src/libstore-tests/data/derivation ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info From c3d4c5f69d93278dafe2c631f955ffe0e47ca689 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 22:00:54 +0000 Subject: [PATCH 122/155] build(deps): bump cachix/install-nix-action from 31.5.1 to 31.8.2 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.5.1 to 31.8.2. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/c134e4c9e34bac6cab09cf239815f9339aaaf84e...456688f15bc354bef6d396e4a35f4f89d40bf2b7) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.8.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67e97b188..60c617978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 + - uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31.8.2 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} From c8e24491c0da15406b736d11f5095a4c504d263c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 3 Nov 2025 14:19:54 -0800 Subject: [PATCH 123/155] Fix warning in kaitai spec Warning: ``` [39/483] Generating src/kaitai-struct-checks/kaitai-generated-sources with a custom command ../src/kaitai-struct-checks/nar.ksy: /types/padded_str/seq/1/encoding: warning: use canonical encoding name `ASCII` instead of `ascii` (see https://doc.kaitai.io/ksy_style_guide.html#encoding-name) ``` --- doc/manual/source/protocols/nix-archive/nar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/protocols/nix-archive/nar.ksy b/doc/manual/source/protocols/nix-archive/nar.ksy index 1cad09097..6a172b276 100644 --- a/doc/manual/source/protocols/nix-archive/nar.ksy +++ b/doc/manual/source/protocols/nix-archive/nar.ksy @@ -29,7 +29,7 @@ types: - id: body type: str size: len_str - encoding: 'ascii' + encoding: 'ASCII' - id: padding size: (8 - (len_str % 8)) % 8 From 389bcba97a1295440a24c887840b1af3e73f0dd3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Oct 2025 13:42:52 -0400 Subject: [PATCH 124/155] JSON Impl and schema for `BuildResult` --- doc/manual/package.nix | 1 + doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/json/build-result.md | 21 +++ doc/manual/source/protocols/json/meson.build | 1 + .../protocols/json/schema/build-result-v1 | 1 + .../json/schema/build-result-v1.yaml | 136 +++++++++++++++++ src/json-schema-checks/build-result | 1 + src/json-schema-checks/meson.build | 9 ++ src/json-schema-checks/package.nix | 1 + src/libstore-tests/build-result.cc | 108 +++++++++++++ .../data/build-result/not-deterministic.json | 9 ++ .../data/build-result/output-rejected.json | 9 ++ .../data/build-result/success.json | 23 +++ src/libstore-tests/meson.build | 1 + src/libstore/build-result.cc | 142 ++++++++++++++++++ .../include/nix/store/build-result.hh | 3 + 16 files changed, 467 insertions(+) create mode 100644 doc/manual/source/protocols/json/build-result.md create mode 120000 doc/manual/source/protocols/json/schema/build-result-v1 create mode 100644 doc/manual/source/protocols/json/schema/build-result-v1.yaml create mode 120000 src/json-schema-checks/build-result create mode 100644 src/libstore-tests/build-result.cc create mode 100644 src/libstore-tests/data/build-result/not-deterministic.json create mode 100644 src/libstore-tests/data/build-result/output-rejected.json create mode 100644 src/libstore-tests/data/build-result/success.json diff --git a/doc/manual/package.nix b/doc/manual/package.nix index e13c6f33d..343e40016 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -41,6 +41,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 580076ece..5be3d6a90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Derivation](protocols/json/derivation.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) + - [Build Result](protocols/json/build-result.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) diff --git a/doc/manual/source/protocols/json/build-result.md b/doc/manual/source/protocols/json/build-result.md new file mode 100644 index 000000000..527e7bcc0 --- /dev/null +++ b/doc/manual/source/protocols/json/build-result.md @@ -0,0 +1,21 @@ +{{#include build-result-v1-fixed.md}} + +## Examples + +### Successful build + +```json +{{#include schema/build-result-v1/success.json}} +``` + +### Failed build (output rejected) + +```json +{{#include schema/build-result-v1/output-rejected.json}} +``` + +### Failed build (non-deterministic) + +```json +{{#include schema/build-result-v1/not-deterministic.json}} +``` \ No newline at end of file diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index d8e94d68c..c56de49c7 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'derivation-v3', 'deriving-path-v1', 'build-trace-entry-v1', + 'build-result-v1', ] schema_files = files() diff --git a/doc/manual/source/protocols/json/schema/build-result-v1 b/doc/manual/source/protocols/json/schema/build-result-v1 new file mode 120000 index 000000000..a143d2c50 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml new file mode 100644 index 000000000..31f59a44d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -0,0 +1,136 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-result-v1.json" +title: Build Result +description: | + This schema describes the JSON representation of Nix's `BuildResult` type, which represents the result of building a derivation or substituting store paths. + + Build results can represent either successful builds (with built outputs) or various types of failures. + +oneOf: + - "$ref": "#/$defs/success" + - "$ref": "#/$defs/failure" +type: object +required: + - success + - status +properties: + timesBuilt: + type: integer + minimum: 0 + title: Times built + description: | + How many times this build was performed. + + startTime: + type: integer + minimum: 0 + title: Start time + description: | + The start time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + stopTime: + type: integer + minimum: 0 + title: Stop time + description: | + The stop time of the build (or one of the rounds, if it was repeated), as a Unix timestamp. + + cpuUser: + type: integer + minimum: 0 + title: User CPU time + description: | + User CPU time the build took, in microseconds. + + cpuSystem: + type: integer + minimum: 0 + title: System CPU time + description: | + System CPU time the build took, in microseconds. + +"$defs": + success: + type: object + title: Successful Build Result + description: | + Represents a successful build with built outputs. + required: + - success + - status + - builtOutputs + properties: + success: + const: true + title: Success indicator + description: | + Always true for successful build results. + + status: + type: string + title: Success status + description: | + Status string for successful builds. + enum: + - "Built" + - "Substituted" + - "AlreadyValid" + - "ResolvesToAlreadyValid" + + builtOutputs: + type: object + title: Built outputs + description: | + A mapping from output names to their build trace entries. + additionalProperties: + "$ref": "build-trace-entry-v1.yaml" + + failure: + type: object + title: Failed Build Result + description: | + Represents a failed build with error information. + required: + - success + - status + - errorMsg + properties: + success: + const: false + title: Success indicator + description: | + Always false for failed build results. + + status: + type: string + title: Failure status + description: | + Status string for failed builds. + enum: + - "PermanentFailure" + - "InputRejected" + - "OutputRejected" + - "TransientFailure" + - "CachedFailure" + - "TimedOut" + - "MiscFailure" + - "DependencyFailed" + - "LogLimitExceeded" + - "NotDeterministic" + - "NoSubstituters" + - "HashMismatch" + + errorMsg: + type: string + title: Error message + description: | + Information about the error if the build failed. + + isNonDeterministic: + type: boolean + title: Non-deterministic flag + description: | + If timesBuilt > 1, whether some builds did not produce the same result. + + Note that 'isNonDeterministic = false' does not mean the build is deterministic, + just that we don't have evidence of non-determinism. diff --git a/src/json-schema-checks/build-result b/src/json-schema-checks/build-result new file mode 120000 index 000000000..8010d0fdd --- /dev/null +++ b/src/json-schema-checks/build-result @@ -0,0 +1 @@ +../../src/libstore-tests/data/build-result \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index c2c7fbff4..65a2651b7 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -150,6 +150,15 @@ schemas += [ 'impure.json', ], }, + { + 'stem' : 'build-result', + 'schema' : schema_dir / 'build-result-v1.yaml', + 'files' : [ + 'success.json', + 'output-rejected.json', + 'not-deterministic.json', + ], + }, # Match exact variant { 'stem' : 'store-object-info', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 057a6e85b..5365fe75e 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -28,6 +28,7 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/derived-path ../../src/libstore-tests/data/path-info ../../src/libstore-tests/data/nar-info + ../../src/libstore-tests/data/build-result ./. ]; diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc new file mode 100644 index 000000000..85e799c2a --- /dev/null +++ b/src/libstore-tests/build-result.cc @@ -0,0 +1,108 @@ +#include + +#include "nix/store/build-result.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class BuildResultTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "build-result"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct BuildResultJsonTest : BuildResultTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(BuildResultJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(BuildResultJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +using namespace std::literals::chrono_literals; + +INSTANTIATE_TEST_SUITE_P( + BuildResultJSON, + BuildResultJsonTest, + ::testing::Values( + std::pair{ + "not-deterministic", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = false, // Note: This field is separate from the status + }}, + .timesBuilt = 1, + }, + }, + std::pair{ + "output-rejected", + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + .isNonDeterministic = false, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + }, + std::pair{ + "success", + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs{ + { + "foo", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + }, + }, + { + "bar", + { + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + DrvOutput{ + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + }, + }, + }, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + })); + +} // namespace nix diff --git a/src/libstore-tests/data/build-result/not-deterministic.json b/src/libstore-tests/data/build-result/not-deterministic.json new file mode 100644 index 000000000..c24a15795 --- /dev/null +++ b/src/libstore-tests/data/build-result/not-deterministic.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "NotDeterministic", + "stopTime": 0, + "success": false, + "timesBuilt": 1 +} diff --git a/src/libstore-tests/data/build-result/output-rejected.json b/src/libstore-tests/data/build-result/output-rejected.json new file mode 100644 index 000000000..9494bf4ec --- /dev/null +++ b/src/libstore-tests/data/build-result/output-rejected.json @@ -0,0 +1,9 @@ +{ + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 30, + "status": "OutputRejected", + "stopTime": 50, + "success": false, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json new file mode 100644 index 000000000..4baadb547 --- /dev/null +++ b/src/libstore-tests/data/build-result/success.json @@ -0,0 +1,23 @@ +{ + "builtOutputs": { + "bar": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "dependentRealisations": {}, + "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 3 +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 4d464ad89..f76df8bcb 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -54,6 +54,7 @@ deps_private += gtest subdir('nix-meson-build-support/common') sources = files( + 'build-result.cc', 'common-protocol.cc', 'content-address.cc', 'derivation-advanced-attrs.cc', diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index ecbd27b49..e3d9e9085 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,4 +1,6 @@ #include "nix/store/build-result.hh" +#include "nix/util/json-utils.hh" +#include namespace nix { @@ -11,4 +13,144 @@ std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Succes bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; +static constexpr std::array, 4> successStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Success::e, #e} + ENUM_ENTRY(Built), + ENUM_ENTRY(Substituted), + ENUM_ENTRY(AlreadyValid), + ENUM_ENTRY(ResolvesToAlreadyValid), +#undef ENUM_ENTRY +}}; + +static std::string_view successStatusToString(BuildResult::Success::Status status) +{ + for (const auto & [enumVal, str] : successStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown success status: %d", static_cast(status)); +} + +static BuildResult::Success::Status successStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : successStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result success status '%s'", str); +} + +static constexpr std::array, 12> failureStatusStrings{{ +#define ENUM_ENTRY(e) {BuildResult::Failure::e, #e} + ENUM_ENTRY(PermanentFailure), + ENUM_ENTRY(InputRejected), + ENUM_ENTRY(OutputRejected), + ENUM_ENTRY(TransientFailure), + ENUM_ENTRY(CachedFailure), + ENUM_ENTRY(TimedOut), + ENUM_ENTRY(MiscFailure), + ENUM_ENTRY(DependencyFailed), + ENUM_ENTRY(LogLimitExceeded), + ENUM_ENTRY(NotDeterministic), + ENUM_ENTRY(NoSubstituters), + ENUM_ENTRY(HashMismatch), +#undef ENUM_ENTRY +}}; + +static std::string_view failureStatusToString(BuildResult::Failure::Status status) +{ + for (const auto & [enumVal, str] : failureStatusStrings) { + if (enumVal == status) + return str; + } + throw Error("unknown failure status: %d", static_cast(status)); +} + +static BuildResult::Failure::Status failureStatusFromString(std::string_view str) +{ + for (const auto & [enumVal, enumStr] : failureStatusStrings) { + if (enumStr == str) + return enumVal; + } + throw Error("unknown built result failure status '%s'", str); +} + } // namespace nix + +namespace nlohmann { + +using namespace nix; + +void adl_serializer::to_json(json & res, const BuildResult & br) +{ + res = json::object(); + + // Common fields + res["timesBuilt"] = br.timesBuilt; + res["startTime"] = br.startTime; + res["stopTime"] = br.stopTime; + + if (br.cpuUser.has_value()) { + res["cpuUser"] = br.cpuUser->count(); + } + if (br.cpuSystem.has_value()) { + res["cpuSystem"] = br.cpuSystem->count(); + } + + // Handle success or failure variant + std::visit( + overloaded{ + [&](const BuildResult::Success & success) { + res["success"] = true; + res["status"] = successStatusToString(success.status); + res["builtOutputs"] = success.builtOutputs; + }, + [&](const BuildResult::Failure & failure) { + res["success"] = false; + res["status"] = failureStatusToString(failure.status); + res["errorMsg"] = failure.errorMsg; + res["isNonDeterministic"] = failure.isNonDeterministic; + }, + }, + br.inner); +} + +BuildResult adl_serializer::from_json(const json & _json) +{ + auto & json = getObject(_json); + + BuildResult br; + + // Common fields + br.timesBuilt = getUnsigned(valueAt(json, "timesBuilt")); + br.startTime = getUnsigned(valueAt(json, "startTime")); + br.stopTime = getUnsigned(valueAt(json, "stopTime")); + + if (auto cpuUser = optionalValueAt(json, "cpuUser")) { + br.cpuUser = std::chrono::microseconds(getUnsigned(*cpuUser)); + } + if (auto cpuSystem = optionalValueAt(json, "cpuSystem")) { + br.cpuSystem = std::chrono::microseconds(getUnsigned(*cpuSystem)); + } + + // Determine success or failure based on success field + bool success = getBoolean(valueAt(json, "success")); + std::string statusStr = getString(valueAt(json, "status")); + + if (success) { + BuildResult::Success s; + s.status = successStatusFromString(statusStr); + s.builtOutputs = valueAt(json, "builtOutputs"); + br.inner = std::move(s); + } else { + BuildResult::Failure f; + f.status = failureStatusFromString(statusStr); + f.errorMsg = getString(valueAt(json, "errorMsg")); + f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")); + br.inner = std::move(f); + } + + return br; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 0446c4038..4739232f8 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -7,6 +7,7 @@ #include "nix/store/derived-path.hh" #include "nix/store/realisation.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -175,3 +176,5 @@ struct KeyedBuildResult : BuildResult }; } // namespace nix + +JSON_IMPL(nix::BuildResult) From 469123eda19028ea784b78b6d21ae0d4e2c91ab3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 14:10:13 +0200 Subject: [PATCH 125/155] doc: Check link fragments with lychee --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a70617b74..897889a71 100644 --- a/flake.nix +++ b/flake.nix @@ -320,7 +320,16 @@ checks = forAllSystems ( system: - (import ./ci/gha/tests { + let + pkgs = nixpkgsFor.${system}.native; + in + { + # https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck + linkcheck = pkgs.testers.lycheeLinkCheck { + site = self.packages.${system}.nix-manual + "/share/doc/nix/manual"; + }; + } + // (import ./ci/gha/tests { inherit system; pkgs = nixpkgsFor.${system}.native; nixFlake = self; From ae15d4eaf395f02a6b08e75044d5b5d48e1cb12c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Nov 2025 00:18:51 +0100 Subject: [PATCH 126/155] Fix links in the manual --- doc/manual/anchors.jq | 2 +- doc/manual/source/command-ref/nix-channel.md | 2 +- .../source/command-ref/nix-env/upgrade.md | 2 +- doc/manual/source/development/building.md | 4 ++-- doc/manual/source/development/testing.md | 2 +- doc/manual/source/glossary.md | 10 +++++----- .../source/language/advanced-attributes.md | 2 +- doc/manual/source/language/derivations.md | 2 +- doc/manual/source/language/identifiers.md | 2 +- doc/manual/source/language/index.md | 10 +++++----- doc/manual/source/language/string-context.md | 2 +- .../source/language/string-interpolation.md | 2 +- doc/manual/source/language/syntax.md | 6 +++--- .../source/protocols/json/schema/hash-v1.yaml | 2 +- .../source/protocols/nix-archive/index.md | 2 +- doc/manual/source/release-notes/rl-2.18.md | 2 +- doc/manual/source/release-notes/rl-2.19.md | 4 ++-- doc/manual/source/release-notes/rl-2.23.md | 2 +- doc/manual/source/release-notes/rl-2.24.md | 2 +- doc/manual/source/store/building.md | 2 +- doc/manual/source/store/derivation/index.md | 2 +- .../source/store/derivation/outputs/index.md | 2 +- .../store/store-object/content-address.md | 4 ++-- doc/manual/source/store/store-path.md | 2 +- src/libexpr/primops.cc | 20 +++++++++---------- src/libexpr/primops/context.cc | 4 ++-- src/libstore/include/nix/store/globals.hh | 2 +- src/libstore/include/nix/store/local-store.hh | 2 +- src/libutil/experimental-features.cc | 2 +- src/nix/flake.md | 2 +- 30 files changed, 53 insertions(+), 53 deletions(-) diff --git a/doc/manual/anchors.jq b/doc/manual/anchors.jq index 72309779c..4ee2bc130 100755 --- a/doc/manual/anchors.jq +++ b/doc/manual/anchors.jq @@ -3,7 +3,7 @@ def transform_anchors_html: - . | gsub($empty_anchor_regex; "") + . | gsub($empty_anchor_regex; "") | gsub($anchor_regex; "" + .text + ""); diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index ed9cbb41f..3d02a7d40 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -14,7 +14,7 @@ The moving parts of channels are: - The official channels listed at - The user-specific list of [subscribed channels](#subscribed-channels) - The [downloaded channel contents](#channels) -- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-I) or the [`NIX_PATH` environment variable](#env-NIX_PATH) > **Note** > diff --git a/doc/manual/source/command-ref/nix-env/upgrade.md b/doc/manual/source/command-ref/nix-env/upgrade.md index 2779363c3..bf4c1a8ed 100644 --- a/doc/manual/source/command-ref/nix-env/upgrade.md +++ b/doc/manual/source/command-ref/nix-env/upgrade.md @@ -22,7 +22,7 @@ left untouched; this is not an error. It is also not an error if an element of *args* matches no installed derivations. For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple +[`--install`](./install.md). If *args* describes multiple store paths with the same symbolic name, only the one with the highest version is installed. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 889d81d80..eb65a7247 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -66,7 +66,7 @@ You can also build Nix for one of the [supported platforms](#platforms). This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. [`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command +[`nix-command`]: @docroot@/development/experimental-features.md#xp-feature-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -256,7 +256,7 @@ You can use any of the other supported environments in place of `nix-cli-ccacheS ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#building-nix-with-flakes) or in [classic Nix](#building-nix). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index c0b130155..7c2cbbb5d 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -119,7 +119,7 @@ This will: 3. Stop the program when the test fails, allowing the user to then issue arbitrary commands to GDB. -### Characterisation testing { #characaterisation-testing-unit } +### Characterisation testing { #characterisation-testing-unit } See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index e6a294e7d..502e6d4de 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -208,7 +208,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -279,7 +279,7 @@ See [References](@docroot@/store/store-object.md#references) for details. -- [referrer]{#gloss-reference} +- [referrer]{#gloss-referrer} A reversed edge from one [store object] to another. @@ -367,8 +367,8 @@ Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#attribute-set) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#output), + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - attributes with metadata, such as information about how the package is supposed to be used. The exact shape of these attribute sets is up to convention. @@ -383,7 +383,7 @@ [string]: ./language/types.md#type-string [path]: ./language/types.md#type-path - [attribute name]: ./language/types.md#attribute-set + [attribute name]: ./language/types.md#type-attrs - [base directory]{#gloss-base-directory} diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index c9d64f060..f0b1a4c73 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -333,7 +333,7 @@ Here is more information on the `output*` attributes, and what values they may b `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format, because in that case the choice of hash algorithm is determined by `outputHash`. - - [`outputHash`]{#adv-attr-outputHashAlgo}; [`outputHash`]{#adv-attr-outputHashMode} + - [`outputHash`]{#adv-attr-outputHash} This will specify the output hash of the single output of a [fixed-output derivation]. diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 43eec680b..2403183fc 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -16,7 +16,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. - See [derivation outputs](@docroot@/store/derivation/index.md#outputs) for what this is affects. + See [derivation outputs](@docroot@/store/derivation/outputs/index.md#outputs) for what this is affects. [store path]: @docroot@/store/store-path.md diff --git a/doc/manual/source/language/identifiers.md b/doc/manual/source/language/identifiers.md index 584a2f861..67bb1eeec 100644 --- a/doc/manual/source/language/identifiers.md +++ b/doc/manual/source/language/identifiers.md @@ -16,7 +16,7 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). +A *name* can be written as an [identifier](#identifiers) or a [string literal](./string-literals.md). > **Syntax** > diff --git a/doc/manual/source/language/index.md b/doc/manual/source/language/index.md index 1eb14e96d..116f928dc 100644 --- a/doc/manual/source/language/index.md +++ b/doc/manual/source/language/index.md @@ -137,7 +137,7 @@ This is an incomplete overview of language features, by example. - [Booleans](@docroot@/language/types.md#type-boolean) + [Booleans](@docroot@/language/types.md#type-bool) @@ -245,7 +245,7 @@ This is an incomplete overview of language features, by example. - An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#type-attrs) with attributes named `x` and `y` @@ -285,7 +285,7 @@ This is an incomplete overview of language features, by example. - [Lists](@docroot@/language/types.md#list) with three elements. + [Lists](@docroot@/language/types.md#type-list) with three elements. @@ -369,7 +369,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#type-attrs) (evaluates to `1`) @@ -381,7 +381,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#type-attrs) with default (evaluates to `3`) diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 0d8fcdefa..65c59d865 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp [`builtins.hasContext`]: ./builtins.md#builtins-hasContext [`builtins.getContext`]: ./builtins.md#builtins-getContext -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs ## Clearing string contexts diff --git a/doc/manual/source/language/string-interpolation.md b/doc/manual/source/language/string-interpolation.md index a503d5f04..8e25d2b63 100644 --- a/doc/manual/source/language/string-interpolation.md +++ b/doc/manual/source/language/string-interpolation.md @@ -6,7 +6,7 @@ Such a construct is called *interpolated string*, and the expression inside is a [string]: ./types.md#type-string [path]: ./types.md#type-path -[attribute set]: ./types.md#attribute-set +[attribute set]: ./types.md#type-attrs > **Syntax** > diff --git a/doc/manual/source/language/syntax.md b/doc/manual/source/language/syntax.md index 85162db74..b127aca14 100644 --- a/doc/manual/source/language/syntax.md +++ b/doc/manual/source/language/syntax.md @@ -51,7 +51,7 @@ See [String literals](string-literals.md). Path literals can also include [string interpolation], besides being [interpolated into other expressions]. - [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expression At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. @@ -235,7 +235,7 @@ of object-oriented programming, for example. ## Recursive sets -Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. +Recursive sets are like normal [attribute sets](./types.md#type-attrs), but the attributes can refer to each other. > *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` @@ -287,7 +287,7 @@ This evaluates to `"foobar"`. ## Inheriting attributes -When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). This can be shortened using the `inherit` keyword. Example: diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml index 316fb6d73..821546dee 100644 --- a/doc/manual/source/protocols/json/schema/hash-v1.yaml +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -51,4 +51,4 @@ additionalProperties: false description: | The hash algorithm used to compute the hash value. - `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature. + `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake3-hashes) experimental feature. diff --git a/doc/manual/source/protocols/nix-archive/index.md b/doc/manual/source/protocols/nix-archive/index.md index 4d25f63e2..bd2a8e833 100644 --- a/doc/manual/source/protocols/nix-archive/index.md +++ b/doc/manual/source/protocols/nix-archive/index.md @@ -4,7 +4,7 @@ This is the complete specification of the [Nix Archive] format. The Nix Archive format closely follows the abstract specification of a [file system object] tree, because it is designed to serialize exactly that data structure. -[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive [file system object]: @docroot@/store/file-system-object.md The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. diff --git a/doc/manual/source/release-notes/rl-2.18.md b/doc/manual/source/release-notes/rl-2.18.md index eb26fc9e7..71b25f408 100644 --- a/doc/manual/source/release-notes/rl-2.18.md +++ b/doc/manual/source/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/language/advanced-attributes.md#adv-attr-unsafeDiscardReferences) attribute is no longer guarded by an experimental flag and can be used freely. diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 06c704324..04f8c9c28 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-feature-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes). - The interface for creating and updating lock files has been overhauled: diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index e6b0e9ffc..b358a0fdc 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index d4af3cb51..e9b46bb22 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -93,7 +93,7 @@ - Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) - Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/conf-file.md#conf-min-free) to set the minimum free space to 1 gigabyte. This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index dbfe6b5ca..f2d470e99 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -8,7 +8,7 @@ - Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. -## Builder Execution +## Builder Execution {#builder-execution} The [`builder`](./derivation/index.md#builder) is executed as follows: diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 61c5335ff..670f3b2bd 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -102,7 +102,7 @@ But rather than somehow scanning all the other fields for inputs, Nix requires t ### System {#system} -The system type on which the [`builder`](#attr-builder) executable is meant to be run. +The system type on which the [`builder`](#builder) executable is meant to be run. A necessary condition for Nix to schedule a given derivation on some [Nix instance] is for the "system" of that derivation to match that instance's [`system` configuration option] or [`extra-platforms` configuration option]. diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index 0683f5703..ca2ce6665 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -43,7 +43,7 @@ In particular, the specification decides: - if the content is content-addressed, how is it content addressed -- if the content is content-addressed, [what is its content address](./content-address.md#fixed-content-addressing) (and thus what is its [store path]) +- if the content is content-addressed, [what is its content address](./content-address.md#fixed) (and thus what is its [store path]) ## Types of derivations diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 36e841fa3..7834ac510 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -1,7 +1,7 @@ # Content-Addressing Store Objects Just [like][fso-ca] [File System Objects][File System Object], -[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-address), unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. @@ -107,7 +107,7 @@ References (to other store objects and self-references alike) are supported so l > > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. -This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. +This uses the corresponding [Git](../file-system-object/content-address.md#git) method of file system object content addressing. References are not supported. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index beec2389b..4061f3653 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -6,7 +6,7 @@ > > A rendered store path -Nix implements references to [store objects](./index.md#store-object) as *store paths*. +Nix implements references to [store objects](./store-object.md) as *store paths*. Think of a store path as an [opaque], [unique identifier]: The only way to obtain store path is by adding or building store objects. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 96e79fedd..d1aae64fa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -825,10 +825,10 @@ static RegisterPrimOp primop_genericClosure( - [Int](@docroot@/language/types.md#type-int) - [Float](@docroot@/language/types.md#type-float) - - [Boolean](@docroot@/language/types.md#type-boolean) + - [Boolean](@docroot@/language/types.md#type-bool) - [String](@docroot@/language/types.md#type-string) - [Path](@docroot@/language/types.md#type-path) - - [List](@docroot@/language/types.md#list) + - [List](@docroot@/language/types.md#type-list) The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. Items are compared by their `key` attribute. @@ -2103,7 +2103,7 @@ static RegisterPrimOp primop_findFile( builtins.findFile builtins.nixPath "nixpkgs" ``` - A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + A search path is represented as a list of [attribute sets](./types.md#type-attrs) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location @@ -2395,7 +2395,7 @@ static RegisterPrimOp primop_outputOf({ returns an input placeholder for the output of the output of `myDrv`. - This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-paths), e.g. as part of installable syntax on the command line. + This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-path), e.g. as part of installable syntax on the command line. )", .fun = prim_outputOf, .experimentalFeature = Xp::DynamicDerivations, @@ -4966,7 +4966,7 @@ static RegisterPrimOp primop_compareVersions({ version *s1* is older than version *s2*, `0` if they are the same, and `1` if *s1* is newer than *s2*. The version comparison algorithm is the same as the one used by [`nix-env - -u`](../command-ref/nix-env.md#operation---upgrade). + -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_compareVersions, }); @@ -4995,7 +4995,7 @@ static RegisterPrimOp primop_splitVersion({ .doc = R"( Split a string representing a version into its components, by the same version splitting logic underlying the version comparison in - [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade). + [`nix-env -u`](../command-ref/nix-env/upgrade.md). )", .fun = prim_splitVersion, }); @@ -5045,9 +5045,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `true` is not special, and can be shadowed: @@ -5068,9 +5068,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) Primitive value. It can be returned by - [comparison operators](@docroot@/language/operators.md#Comparison) + [comparison operators](@docroot@/language/operators.md#comparison) and used in - [conditional expressions](@docroot@/language/syntax.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#conditionals). The name `false` is not special, and can be shadowed: diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 12b8ffdf9..8a9fe42e8 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -79,7 +79,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency( Create a copy of the given string where every [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element is turned into a - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element. This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). @@ -145,7 +145,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies( .args = {"s"}, .doc = R"( Create a copy of the given string where a single - [constant](@docroot@/language/string-context.md#string-context-element-constant) + [constant](@docroot@/language/string-context.md#string-context-constant) string context element is turned into a [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) string context element. diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 8aa82c4a2..5ddfbee30 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -189,7 +189,7 @@ public: 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism. diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 27895d42a..9f7b750da 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -11,6 +11,7 @@ s/\\`/`/g # # As we have more such relative links, more replacements of this nature # should appear below. +s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c56de49c7..fcc0e5ece 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false) json_schema_config = files('json-schema-for-humans-config.yaml') schemas = [ + 'file-system-object-v1', 'hash-v1', 'content-address-v1', 'store-path-v1', diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1 b/doc/manual/source/protocols/json/schema/file-system-object-v1 new file mode 120000 index 000000000..cbb21a10d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1 @@ -0,0 +1 @@ +../../../../../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml new file mode 100644 index 000000000..c7154b18d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml @@ -0,0 +1,65 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json +title: File System Object +description: | + This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md). + + The schema is recursive because file system objects contain other file system objects. +type: object +required: ["type"] +properties: + type: + type: string + enum: ["regular", "symlink", "directory"] + +# Enforce conditional structure based on `type` +anyOf: + - $ref: "#/$defs/regular" + required: ["type", "contents"] + + - $ref: "#/$defs/symlink" + required: ["type", "target"] + + - $ref: "#/$defs/directory" + required: ["type", "contents"] + +"$defs": + regular: + title: Regular File + required: ["contents"] + properties: + type: + const: "regular" + contents: + type: string + description: Base64-encoded file contents + executable: + type: boolean + description: Whether the file is executable. + default: false + additionalProperties: false + + symlink: + title: Symbolic Link + required: ["target"] + properties: + type: + const: "symlink" + target: + type: string + description: Target path of the symlink. + additionalProperties: false + + directory: + title: Directory + required: ["contents"] + properties: + type: + const: "directory" + contents: + type: object + description: | + Map of names to nested file system objects (for type=directory) + additionalProperties: + $ref: "#" + additionalProperties: false diff --git a/src/json-schema-checks/file-system-object b/src/json-schema-checks/file-system-object new file mode 120000 index 000000000..b26e030c9 --- /dev/null +++ b/src/json-schema-checks/file-system-object @@ -0,0 +1 @@ +../../src/libutil-tests/data/memory-source-accessor \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 65a2651b7..d31ba3d53 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema' # Get all example files schemas = [ + { + 'stem' : 'file-system-object', + 'schema' : schema_dir / 'file-system-object-v1.yaml', + 'files' : [ + 'simple.json', + 'complex.json', + ], + }, { 'stem' : 'hash', 'schema' : schema_dir / 'hash-v1.yaml', diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix index 5365fe75e..2ca43439f 100644 --- a/src/json-schema-checks/package.nix +++ b/src/json-schema-checks/package.nix @@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: { fileset = lib.fileset.unions [ ../../.version ../../doc/manual/source/protocols/json/schema + ../../src/libutil-tests/data/memory-source-accessor ../../src/libutil-tests/data/hash ../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/store-path diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 9cecd573e..e23b443fb 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -1,5 +1,6 @@ #include "nix/store/references.hh" #include "nix/store/path-references.hh" +#include "nix/util/bytes.hh" #include "nix/util/memory-source-accessor.hh" #include @@ -104,21 +105,21 @@ TEST(references, scanForReferencesDeep) // file1.txt: contains hash1 "file1.txt", File::Regular{ - .contents = "This file references " + hash1 + " in its content", + .contents = to_owned(as_bytes("This file references " + hash1 + " in its content")), }, }, { // file2.txt: contains hash2 and hash3 "file2.txt", File::Regular{ - .contents = "Multiple refs: " + hash2 + " and also " + hash3, + .contents = to_owned(as_bytes("Multiple refs: " + hash2 + " and also " + hash3)), }, }, { // file3.txt: contains no references "file3.txt", File::Regular{ - .contents = "This file has no store path references at all", + .contents = to_owned(as_bytes("This file has no store path references at all")), }, }, { @@ -130,7 +131,7 @@ TEST(references, scanForReferencesDeep) // subdir/file4.txt: contains hash1 again "file4.txt", File::Regular{ - .contents = "Subdirectory file with " + hash1, + .contents = to_owned(as_bytes("Subdirectory file with " + hash1)), }, }, }, @@ -140,7 +141,7 @@ TEST(references, scanForReferencesDeep) // link1: a symlink that contains a reference in its target "link1", File::Symlink{ - .target = hash2 + "-target", + .target = to_owned(as_bytes(hash2 + "-target")), }, }, }, diff --git a/src/libutil-tests/data/memory-source-accessor/complex.json b/src/libutil-tests/data/memory-source-accessor/complex.json new file mode 100644 index 000000000..23c2e5b42 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/complex.json @@ -0,0 +1,24 @@ +{ + "contents": { + "bar": { + "contents": { + "baz": { + "contents": "Z29vZCBkYXksCg==", + "executable": true, + "type": "regular" + }, + "quux": { + "target": "L292ZXIvdGhlcmU=", + "type": "symlink" + } + }, + "type": "directory" + }, + "foo": { + "contents": "aGVsbG8K", + "executable": false, + "type": "regular" + } + }, + "type": "directory" +} diff --git a/src/libutil-tests/data/memory-source-accessor/simple.json b/src/libutil-tests/data/memory-source-accessor/simple.json new file mode 100644 index 000000000..8c61fa730 --- /dev/null +++ b/src/libutil-tests/data/memory-source-accessor/simple.json @@ -0,0 +1,5 @@ +{ + "contents": "YXNkZg==", + "executable": false, + "type": "regular" +} diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc..f761c4433 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write) }); } +namespace memory_source_accessor { + +extern ref exampleComplex(); + +} + TEST_F(GitTest, both_roundrip) { - using File = MemorySourceAccessor::File; - - auto files = make_ref(); - files->root = File::Directory{ - .contents{ - { - "foo", - File::Regular{ - .contents = "hello\n\0\n\tworld!", - }, - }, - { - "bar", - File::Directory{ - .contents = - { - { - "baz", - File::Regular{ - .executable = true, - .contents = "good day,\n\0\n\tworld!", - }, - }, - { - "quux", - File::Symlink{ - .target = "/over/there", - }, - }, - }, - }, - }, - }, - }; + auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { std::map cas; diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc new file mode 100644 index 000000000..7227c9fa4 --- /dev/null +++ b/src/libutil-tests/memory-source-accessor.cc @@ -0,0 +1,116 @@ +#include + +#include "nix/util/bytes.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +namespace memory_source_accessor { + +using File = MemorySourceAccessor::File; + +ref exampleSimple() +{ + auto sc = make_ref(); + sc->root = File{File::Regular{ + .executable = false, + .contents = to_owned(as_bytes("asdf")), + }}; + return sc; +} + +ref exampleComplex() +{ + auto files = make_ref(); + files->root = File::Directory{ + .contents{ + { + "foo", + File::Regular{ + .contents = to_owned(as_bytes("hello\n\0\n\tworld!")), + }, + }, + { + "bar", + File::Directory{ + .contents = + { + { + "baz", + File::Regular{ + .executable = true, + .contents = to_owned(as_bytes("good day,\n\0\n\tworld!")), + }, + }, + { + "quux", + File::Symlink{ + .target = to_owned(as_bytes("/over/there")), + }, + }, + }, + }, + }, + }, + }; + return files; +} + +} // namespace memory_source_accessor + +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +class MemorySourceAccessorTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +using nlohmann::json; + +struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(MemorySourceAccessorJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + /* Cannot use `readJsonTest` because need to compare `root` field of + the source accessors for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + auto decoded = static_cast(encoded); + ASSERT_EQ(decoded.root, expected.root); + }); +} + +TEST_P(MemorySourceAccessorJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + MemorySourceAccessorJSON, + MemorySourceAccessorJsonTest, + ::testing::Values( + std::pair{ + "simple", + *memory_source_accessor::exampleSimple(), + }, + std::pair{ + "complex", + *memory_source_accessor::exampleComplex(), + })); + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index c75f4d90a..6c593eef8 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -63,6 +63,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nix_api_util.cc', 'nix_api_util_internal.cc', diff --git a/src/libutil/include/nix/util/bytes.hh b/src/libutil/include/nix/util/bytes.hh new file mode 100644 index 000000000..7fd6cf8c3 --- /dev/null +++ b/src/libutil/include/nix/util/bytes.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +static inline std::span as_bytes(std::string_view sv) noexcept +{ + return std::span{ + reinterpret_cast(sv.data()), + sv.size(), + }; +} + +static inline std::vector to_owned(std::span bytes) +{ + return std::vector{ + bytes.begin(), + bytes.end(), + }; +} + +/** + * @note this should be avoided, as arbitrary binary data in strings + * views, while allowed, is not really proper. Generally this should + * only be used as a stop-gap with other definitions that themselves + * should be converted to accept `std::span` or + * similar, directly. + */ +static inline std::string_view as_str(std::span sp) +{ + return std::string_view{ + reinterpret_cast(sp.data()), + sp.size(), + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1..cd8b3c717 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,6 +4,7 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -25,7 +26,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Regular { bool executable = false; - std::string contents; + std::vector contents; bool operator==(const Regular &) const = default; auto operator<=>(const Regular &) const = default; @@ -44,7 +45,7 @@ struct MemorySourceAccessor : virtual SourceAccessor struct Symlink { - std::string target; + std::vector target; bool operator==(const Symlink &) const = default; auto operator<=>(const Symlink &) const = default; @@ -86,7 +87,13 @@ struct MemorySourceAccessor : virtual SourceAccessor */ File * open(const CanonPath & path, std::optional create); - SourcePath addFile(CanonPath path, std::string && contents); + SourcePath addFile(CanonPath path, std::vector && contents); + + /** + * Small wrapper of the other `addFile`, purely for convenience when + * the file in question to be added is a string. + */ + SourcePath addFile(CanonPath path, std::string_view contents); }; inline bool MemorySourceAccessor::File::Directory::operator==( @@ -121,4 +128,30 @@ struct MemorySink : FileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override; }; +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(MemorySourceAccessor::File::Regular) +JSON_IMPL(MemorySourceAccessor::File::Directory) +JSON_IMPL(MemorySourceAccessor::File::Symlink) +JSON_IMPL(MemorySourceAccessor::File) +JSON_IMPL(MemorySourceAccessor) diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15d..89790f908 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -12,6 +12,7 @@ headers = files( 'array-from-string-literal.hh', 'base-n.hh', 'base-nix-32.hh', + 'bytes.hh', 'callback.hh', 'canon-path.hh', 'checked-arithmetic.hh', diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb7746..5089470e9 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,4 +1,7 @@ #include "nix/util/memory-source-accessor.hh" +#include "nix/util/base-n.hh" +#include "nix/util/bytes.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -58,7 +61,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * r = std::get_if(&f->raw)) - return r->contents; + return std::string{as_str(r->contents)}; else throw Error("file '%s' is not a regular file", path); } @@ -120,12 +123,12 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) if (!f) throw Error("file '%s' does not exist", path); if (auto * s = std::get_if(&f->raw)) - return s->target; + return std::string{as_str(s->target)}; else throw Error("file '%s' is not a symbolic link", path); } -SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector && contents) { // Create root directory automatically if necessary as a convenience. if (!root && !path.isRoot()) @@ -142,6 +145,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents return SourcePath{ref(shared_from_this()), path}; } +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents) +{ + return addFile(path, to_owned(as_bytes(contents))); +} + using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) @@ -190,9 +198,10 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len) regularFile.contents.reserve(len); } -void CreateMemoryRegularFile::operator()(std::string_view data) +void CreateMemoryRegularFile::operator()(std::string_view data_) { - regularFile.contents += data; + auto data = as_bytes(data_); + regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end()); } void MemorySink::createSymlink(const CanonPath & path, const std::string & target) @@ -201,7 +210,7 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * s = std::get_if(&f->raw)) - s->target = target; + s->target = to_owned(as_bytes(target)); else throw Error("file '%s' is not a symbolic link", path); } @@ -222,3 +231,106 @@ ref makeEmptySourceAccessor() } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +MemorySourceAccessor::File::Regular adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Regular{ + .executable = getBoolean(valueAt(obj, "executable")), + .contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Regular & val) +{ + json = { + {"executable", val.executable}, + {"contents", base64::encode(val.contents)}, + }; +} + +MemorySourceAccessor::File::Directory +adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Directory{ + .contents = valueAt(obj, "contents"), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Directory & val) +{ + json = { + {"contents", val.contents}, + }; +} + +MemorySourceAccessor::File::Symlink adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return MemorySourceAccessor::File::Symlink{ + .target = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "target"))))), + }; +} + +void adl_serializer::to_json( + json & json, const MemorySourceAccessor::File::Symlink & val) +{ + json = { + {"target", base64::encode(val.target)}, + }; +} + +MemorySourceAccessor::File adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + auto type = getString(valueAt(obj, "type")); + if (type == "regular") + return static_cast(json); + if (type == "directory") + return static_cast(json); + if (type == "symlink") + return static_cast(json); + else + throw Error("unknown type of file '%s'", type); +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor::File & val) +{ + std::visit( + overloaded{ + [&](const MemorySourceAccessor::File::Regular & r) { + json = r; + json["type"] = "regular"; + }, + [&](const MemorySourceAccessor::File::Directory & d) { + json = d; + json["type"] = "directory"; + }, + [&](const MemorySourceAccessor::File::Symlink & s) { + json = s; + json["type"] = "symlink"; + }, + }, + val.raw); +} + +MemorySourceAccessor adl_serializer::from_json(const json & json) +{ + MemorySourceAccessor res; + res.root = json; + return res; +} + +void adl_serializer::to_json(json & json, const MemorySourceAccessor & val) +{ + json = val.root; +} + +} // namespace nlohmann From f7dcfd107253ec0b1e8ec8700012c739a0761e07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Sep 2025 02:15:20 -0400 Subject: [PATCH 152/155] JSON impl for `DummyStore` --- .../data/dummy-store/empty.json | 6 ++ .../data/dummy-store/one-derivation.json | 18 ++++ .../data/dummy-store/one-flat-file.json | 35 +++++++ src/libstore-tests/dummy-store.cc | 93 ++++++++++++++++++ src/libstore/dummy-store.cc | 97 +++++++++++++++++++ .../include/nix/store/dummy-store-impl.hh | 10 ++ src/libstore/include/nix/store/dummy-store.hh | 7 ++ 7 files changed, 266 insertions(+) create mode 100644 src/libstore-tests/data/dummy-store/empty.json create mode 100644 src/libstore-tests/data/dummy-store/one-derivation.json create mode 100644 src/libstore-tests/data/dummy-store/one-flat-file.json diff --git a/src/libstore-tests/data/dummy-store/empty.json b/src/libstore-tests/data/dummy-store/empty.json new file mode 100644 index 000000000..976a05843 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/empty.json @@ -0,0 +1,6 @@ +{ + "build-trace": {}, + "contents": {}, + "derivations": {}, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-derivation.json b/src/libstore-tests/data/dummy-store/one-derivation.json new file mode 100644 index 000000000..201418db6 --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-derivation.json @@ -0,0 +1,18 @@ +{ + "build-trace": {}, + "contents": {}, + "derivations": { + "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": { + "args": [], + "builder": "", + "env": {}, + "inputDrvs": {}, + "inputSrcs": [], + "name": "foo", + "outputs": {}, + "system": "", + "version": 4 + } + }, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json new file mode 100644 index 000000000..d98990baa --- /dev/null +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -0,0 +1,35 @@ +{ + "build-trace": {}, + "contents": { + "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": { + "contents": { + "contents": "asdf", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "method": "nar" + }, + "deriver": null, + "narHash": { + "algorithm": "sha256", + "format": "base64", + "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU=" + }, + "narSize": 120, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false + } + } + }, + "derivations": {}, + "store-dir": "/nix/store" +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 3dd8137a3..36a0c3019 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,11 +1,32 @@ #include +#include +#include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" #include "nix/store/realisation.hh" +#include "nix/util/tests/json-characterization.hh" + namespace nix { +class DummyStoreTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "dummy-store"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); @@ -35,4 +56,76 @@ TEST(DummyStore, realisation_read) EXPECT_EQ(*value2, value); } +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct DummyStoreJsonTest : DummyStoreTest, + JsonCharacterizationTest>, + ::testing::WithParamInterface>> +{}; + +TEST_P(DummyStoreJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + using namespace nlohmann; + /* Cannot use `readJsonTest` because need to dereference the stores + for equality. */ + readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + ref decoded = adl_serializer>::from_json(encoded); + ASSERT_EQ(*decoded, *expected); + }); +} + +TEST_P(DummyStoreJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { + initLibStore(false); + auto writeCfg = make_ref(DummyStore::Config::Params{}); + writeCfg->readOnly = false; + return ::testing::Values( + std::pair{ + "empty", + make_ref(DummyStore::Config::Params{})->openDummyStore(), + }, + std::pair{ + "one-flat-file", + [&] { + auto store = writeCfg->openDummyStore(); + store->addToStore( + "my-file", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "asdf", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + return store; + }(), + }, + std::pair{ + "one-derivation", + [&] { + auto store = writeCfg->openDummyStore(); + Derivation drv; + drv.name = "foo"; + store->writeDerivation(drv); + return store; + }(), + }); +}()); + } // namespace nix diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c45a13cc3..218a4e2ed 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -2,6 +2,7 @@ #include "nix/util/archive.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/json-utils.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/realisation.hh" @@ -16,6 +17,16 @@ std::string DummyStoreConfig::doc() ; } +bool DummyStore::PathInfoAndContents::operator==(const PathInfoAndContents & other) const +{ + return info == other.info && contents->root == other.contents->root; +} + +bool DummyStore::operator==(const DummyStore & other) const +{ + return contents == other.contents && buildTrace == other.buildTrace; +} + namespace { class WholeStoreViewAccessor : public SourceAccessor @@ -377,3 +388,89 @@ ref DummyStore::Config::openDummyStore() const static RegisterStoreImplementation regDummyStore; } // namespace nix + +namespace nlohmann { + +using namespace nix; + +DummyStore::PathInfoAndContents adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return DummyStore::PathInfoAndContents{ + .info = valueAt(obj, "info"), + .contents = make_ref(valueAt(obj, "contents")), + }; +} + +void adl_serializer::to_json(json & json, const DummyStore::PathInfoAndContents & val) +{ + json = { + {"info", val.info}, + {"contents", *val.contents}, + }; +} + +ref adl_serializer>::from_json(const json & json) +{ + auto & obj = getObject(json); + ref res = [&] { + auto cfg = make_ref(DummyStore::Config::Params{}); + const_cast(cfg->storeDir_).set(getString(valueAt(obj, "store-dir"))); + cfg->readOnly = true; + return cfg->openDummyStore(); + }(); + for (auto & [k, v] : getObject(valueAt(obj, "contents"))) + res->contents.insert({StorePath{k}, v}); + for (auto & [k, v] : getObject(valueAt(obj, "derivations"))) + res->derivations.insert({StorePath{k}, v}); + for (auto & [k0, v] : getObject(valueAt(obj, "build-trace"))) { + for (auto & [k1, v2] : getObject(v)) { + auto vref = make_ref(v2); + res->buildTrace.insert_or_visit( + { + Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + {{k1, vref}}, + }, + [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); + } + } + return res; +} + +void adl_serializer>::to_json(json & json, const ref & val) +{ + json = { + {"store-dir", val->storeDir}, + {"contents", + [&] { + auto obj = json::object(); + val->contents.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"derivations", + [&] { + auto obj = json::object(); + val->derivations.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + obj[k.to_string()] = v; + }); + return obj; + }()}, + {"build-trace", + [&] { + auto obj = json::object(); + val->buildTrace.cvisit_all([&](const auto & kv) { + auto & [k, v] = kv; + auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + for (auto & [k2, v2] : kv.second) + obj2[k2] = *v2; + }); + return obj; + }()}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 137f81c9b..71869cb7e 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -23,6 +23,8 @@ struct DummyStore : virtual Store { UnkeyedValidPathInfo info; ref contents; + + bool operator==(const PathInfoAndContents &) const; }; /** @@ -54,6 +56,14 @@ struct DummyStore : virtual Store , config(config) { } + + bool operator==(const DummyStore &) const; }; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::DummyStore::PathInfoAndContents) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index d371c4e51..6b7b20153 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/json-impls.hh" #include @@ -65,4 +66,10 @@ struct DummyStoreConfig : public std::enable_shared_from_this, } }; +template<> +struct json_avoids_null> : std::true_type +{}; + } // namespace nix + +JSON_IMPL(nix::ref) From f39266353379d4c791a4da4a1ee8c1c098d85f5c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Oct 2025 18:35:19 -0400 Subject: [PATCH 153/155] Remove dependent realisations This progress on #11896. It introduces some issues temporarily which will be fixed when #11928 is fixed. --- .../include/nix/store/tests/protocol.hh | 22 +++-- src/libstore-tests/common-protocol.cc | 50 +++++------ src/libstore-tests/realisation.cc | 81 ++++++++--------- src/libstore-tests/serve-protocol.cc | 12 +-- src/libstore-tests/worker-protocol.cc | 12 +-- src/libstore/build/derivation-goal.cc | 5 -- .../build/drv-output-substitution-goal.cc | 24 ------ src/libstore/ca-specific-schema.sql | 27 ------ src/libstore/include/nix/store/realisation.hh | 16 ---- src/libstore/include/nix/store/store-api.hh | 3 - src/libstore/local-store.cc | 49 ----------- src/libstore/misc.cc | 59 ------------- src/libstore/realisation.cc | 86 +------------------ src/libstore/restricted-store.cc | 2 +- src/libstore/store-api.cc | 40 ++++----- .../ca/duplicate-realisation-in-closure.sh | 5 ++ tests/functional/ca/substitute.sh | 5 +- 17 files changed, 106 insertions(+), 392 deletions(-) diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 0f774df0e..ccb33b17b 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -88,16 +88,22 @@ public: } }; -#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_read) \ - { \ - readProtoTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_write) \ - { \ - writeProtoTest(STEM, VERSION, VALUE); \ +#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_read) \ + { \ + readProtoTest(STEM, VERSION, VALUE); \ } +#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_write) \ + { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) + #define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME##_json_read) \ diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index fa676eb7f..6bd7beb44 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -47,24 +47,30 @@ public: } }; -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(CommonProtoTest, NAME##_read) \ - { \ - readProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_write) \ - { \ - writeProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_read) \ - { \ - readJsonTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_write) \ - { \ - writeJsonTest(STEM, VALUE); \ +#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_read) \ + { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ } +#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_write) \ + { \ + writeProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ + } + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) + CHARACTERIZATION_TEST( string, "string", @@ -141,7 +147,7 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +READ_CHARACTERIZATION_TEST( realisation_with_deps, "realisation-with-deps", (std::tuple{ @@ -149,16 +155,6 @@ CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d16049bc5..66f4707e9 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } +Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }, + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, - ([] { - Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - }; - return ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }()}, - std::pair{ - "with-dependent-realisations", - [&] { - auto r = simple; - r.dependentRealisations = {{ - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }}; - return r; - }(), - }); - } + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + })); - ())); +/** + * We no longer have a notion of "dependent realisations", but we still + * want to parse old realisation files. So make this just be a read test + * (no write direction), accordingly. + */ +TEST_F(RealisationTest, dependent_realisations_from_json) +{ + readJsonTest("with-dependent-realisations", simple); +} } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 8d26c8241..afc35d03c 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -116,7 +116,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, realisation_with_deps, "realisation-with-deps", @@ -126,16 +126,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index f1b639c77..a1f68e231 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -169,7 +169,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, realisation_with_deps, "realisation-with-deps", @@ -179,16 +179,6 @@ VERSIONED_CHARACTERIZATION_TEST( { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, .signatures = {"asdf", "qwer"}, - .dependentRealisations = - { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 14aa044ea..d8e9ff5e4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -209,11 +209,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) .outputName = wantedOutput, }}; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store; - newRealisation.dependentRealisations = - drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } worker.store.signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 8d0a307be..58f3de2b7 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -86,32 +86,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() if (!outputInfo) continue; - bool failed = false; - Goals waitees; - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->config.getHumanReadableURI(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath)); - failed = true; - break; - } - waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - - if (failed) - continue; - waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); co_await await(std::move(waitees)); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e3897..d563b33d8 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -12,30 +12,3 @@ create table if not exists Realisations ( ); create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict -); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index af0e4aefd..3691c14fb 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -56,14 +56,6 @@ struct UnkeyedRealisation StringSet signatures; - /** - * The realisations that are required for the current one to be valid. - * - * When importing this realisation, the store will first check that all its - * dependencies exist, and map to the correct output path - */ - std::map dependentRealisations; - std::string fingerprint(const DrvOutput & key) const; void sign(const DrvOutput & key, const Signer &); @@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation bool isCompatibleWith(const UnkeyedRealisation & other) const; - static std::set closure(Store &, const std::set &); - - static void closure(Store &, const std::set &, std::set & res); - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -154,10 +142,6 @@ struct RealisedPath */ const StorePath & path() const &; - void closure(Store & store, Set & ret) const; - static void closure(Store & store, const Set & startPaths, Set & ret); - Set closure(Store & store) const; - bool operator==(const RealisedPath &) const = default; auto operator<=>(const RealisedPath &) const = default; }; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 4c0b156fa..f272b8588 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1001,9 +1001,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); - template<> struct json_avoids_null : std::true_type {}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6aeaf0d2..98f2d70af 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -390,21 +390,6 @@ LocalStore::LocalStore(ref config) where drvPath = ? ; )"); - state->stmts->QueryRealisationReferences.create( - state->db, - R"( - select drvPath, outputName from Realisations - join RealisationsRefs on realisationReference = Realisations.id - where referrer = ?; - )"); - state->stmts->AddRealisationReference.create( - state->db, - R"( - insert or replace into RealisationsRefs (referrer, realisationReference) - values ( - (select id from Realisations where drvPath = ? and outputName = ?), - (select id from Realisations where drvPath = ? and outputName = ?)); - )"); } } @@ -654,25 +639,6 @@ void LocalStore::registerDrvOutput(const Realisation & info) concatStringsSep(" ", info.signatures)) .exec(); } - for (auto & [outputId, depPath] : info.dependentRealisations) { - auto localRealisation = queryRealisationCore_(*state, outputId); - if (!localRealisation) - throw Error( - "unable to register the derivation '%s' as it " - "depends on the non existent '%s'", - info.id.to_string(), - outputId.to_string()); - if (localRealisation->second.outPath != depPath) - throw Error( - "unable to register the derivation '%s' as it " - "depends on a realisation of '%s' that doesn’t" - "match what we have locally", - info.id.to_string(), - outputId.to_string()); - state->stmts->AddRealisationReference - .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName) - .exec(); - } }); } @@ -1609,21 +1575,6 @@ std::optional LocalStore::queryRealisation_(LocalStore return std::nullopt; auto [realisationDbId, res] = *maybeCore; - std::map dependentRealisations; - auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId)); - while (useRealisationRefs.next()) { - auto depId = DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - }; - auto dependentRealisation = queryRealisationCore_(state, depId); - assert(dependentRealisation); // Enforced by the db schema - auto outputPath = dependentRealisation->second.outPath; - dependentRealisations.insert({depId, outputPath}); - } - - res.dependentRealisations = dependentRealisations; - return {res}; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ea0c8a3c2..6829563c2 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -330,65 +330,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } -std::map -drvOutputReferences(const std::set & inputRealisations, const StorePathSet & pathReferences) -{ - std::map res; - - for (const auto & input : inputRealisations) { - if (pathReferences.count(input.outPath)) { - res.insert({input.id, input.outPath}); - } - } - - return res; -} - -std::map -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_) -{ - auto & evalStore = evalStore_ ? *evalStore_ : store; - - std::set inputRealisations; - - auto accumRealisations = [&](this auto & self, - const StorePath & inputDrv, - const DerivedPathMap::ChildNode & inputNode) -> void { - if (!inputNode.value.empty()) { - auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); - for (const auto & outputName : inputNode.value) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv)); - DrvOutput key{*outputHash, outputName}; - auto thisRealisation = store.queryRealisation(key); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv)); - inputRealisations.insert({*thisRealisation, std::move(key)}); - } - } - if (!inputNode.value.empty()) { - auto d = makeConstantStorePathRef(inputDrv); - for (const auto & [outputName, childNode] : inputNode.childMap) { - SingleDerivedPath next = SingleDerivedPath::Built{d, outputName}; - self( - // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next, evalStore_), - childNode); - } - } - }; - - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) - accumRealisations(inputDrv, inputNode); - - auto info = store.queryPathInfo(outputPath); - - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); -} - OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 4aeb05874..2075b39e6 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,5 @@ #include "nix/store/realisation.hh" #include "nix/store/store-api.hh" -#include "nix/util/closure.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/json-utils.hh" #include @@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const return strHash() + "!" + outputName; } -std::set Realisation::closure(Store & store, const std::set & startOutputs) -{ - std::set res; - Realisation::closure(store, startOutputs, res); - return res; -} - -void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) -{ - auto getDeps = [&](const Realisation & current) -> std::set { - std::set res; - for (auto & [currentDep, _] : current.dependentRealisations) { - if (auto currentRealisation = store.queryRealisation(currentDep)) - res.insert({*currentRealisation, currentDep}); - else - throw Error("Unrealised derivation '%s'", currentDep.to_string()); - } - return res; - }; - - computeClosure( - startOutputs, - res, - [&](const Realisation & current, std::function> &)> processEdges) { - std::promise> promise; - try { - auto res = getDeps(current); - promise.set_value(res); - } catch (...) { - promise.set_exception(std::current_exception()); - } - return processEdges(promise); - }); -} - std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { nlohmann::json serialized = Realisation{*this, key}; @@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const & bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const { - if (outPath == other.outPath) { - if (dependentRealisations.empty() != other.dependentRealisations.empty()) { - warn( - "Encountered a realisation for '%s' with an empty set of " - "dependencies. This is likely an artifact from an older Nix. " - "I’ll try to fix the realisation if I can", - id.to_string()); - return true; - } else if (dependentRealisations == other.dependentRealisations) { - return true; - } - } - return false; -} - -void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret) -{ - // FIXME: This only builds the store-path closure, not the real realisation - // closure - StorePathSet initialStorePaths, pathsClosure; - for (auto & path : startPaths) - initialStorePaths.insert(path.path()); - store.computeFSClosure(initialStorePaths, pathsClosure); - ret.insert(startPaths.begin(), startPaths.end()); - ret.insert(pathsClosure.begin(), pathsClosure.end()); -} - -void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const -{ - RealisedPath::closure(store, {*this}, ret); -} - -RealisedPath::Set RealisedPath::closure(Store & store) const -{ - RealisedPath::Set ret; - closure(store, ret); - return ret; + return outPath == other.outPath; } } // namespace nix @@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer::from_json(const json & js if (auto signaturesOpt = optionalValueAt(json, "signatures")) signatures = *signaturesOpt; - std::map dependentRealisations; - if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) - for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies)) - dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath}); - return UnkeyedRealisation{ .outPath = valueAt(json, "outPath"), .signatures = signatures, - .dependentRealisations = dependentRealisations, }; } void adl_serializer::to_json(json & json, const UnkeyedRealisation & r) { - auto jsonDependentRealisations = nlohmann::json::object(); - for (auto & [depId, depOutPath] : r.dependentRealisations) - jsonDependentRealisations.emplace(depId.to_string(), depOutPath); json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - {"dependentRealisations", jsonDependentRealisations}, + // back-compat + {"dependentRealisations", json::object()}, }; } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index ef8aaa380..b4d696a53 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -292,7 +292,7 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : Realisation::closure(*next, newRealisations)) + for (auto & real : newRealisations) goal.addedDrvOutputs.insert(real.id); return results; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c292e2e43..c70af9301 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -915,36 +915,21 @@ std::map copyPaths( SubstituteFlag substitute) { StorePathSet storePaths; - std::set toplevelRealisations; + std::vector realisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto * realisation = std::get_if(&path.raw)) { experimentalFeatureSettings.require(Xp::CaDerivations); - toplevelRealisations.insert(*realisation); + realisations.push_back(realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - // Copy the realisation closure - processGraph( - Realisation::closure(srcStore, toplevelRealisations), - [&](const Realisation & current) -> std::set { - std::set children; - for (const auto & [drvOutput, _] : current.dependentRealisations) { - auto currentChild = srcStore.queryRealisation(drvOutput); - if (!currentChild) - throw Error( - "incomplete realisation closure: '%s' is a " - "dependency of '%s' but isn't registered", - drvOutput.to_string(), - current.id.to_string()); - children.insert({*currentChild, drvOutput}); - } - return children; - }, - [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); }); + // Copy the realisations. TODO batch this + for (const auto * realisation : realisations) + dstStore.registerDrvOutput(*realisation, checkSigs); } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want @@ -1055,8 +1040,19 @@ void copyClosure( if (&srcStore == &dstStore) return; - RealisedPath::Set closure; - RealisedPath::closure(srcStore, paths, closure); + StorePathSet closure0; + for (auto & path : paths) { + if (auto * opaquePath = std::get_if(&path.raw)) { + closure0.insert(opaquePath->path); + } + } + + StorePathSet closure1; + srcStore.computeFSClosure(closure0, closure1); + + RealisedPath::Set closure = paths; + for (auto && path : closure1) + closure.insert({std::move(path)}); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index 4a5e8c042..032fb6164 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -25,4 +25,9 @@ nix build -f nondeterministic.nix dep2 --no-link # If everything goes right, we should rebuild dep2 rather than fetch it from # the cache (because that would mean duplicating `current-time` in the closure), # and have `dep1 == dep2`. + +# FIXME: Force the use of small-step resolutions only to fix this in a +# better way (#11896, #11928). +skipTest "temporarily broken because dependent realisations are removed" + nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 9728470f0..2f6ebcef5 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -22,7 +22,10 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA +# FIXME: `dependentCA` should not need to be explicitly mentioned in +# this. Force the use of small-step resolutions only to allow not +# mentioning it explicitly again. (#11896, #11928). +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too From b41ce0d263ef4e50181ebb45b3f5dd708e77193c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Feb 2025 23:54:24 -0500 Subject: [PATCH 154/155] Revert "Use the hash modulo in the derivation outputs" Fix #11897 As described in the issue, this makes for a simpler and much more intuitive notion of a realisation key. This is better for pedagogy, and interoperability between more tools. The way the issue was written was that we would switch to only having shallow realisations first, and then do this. But going to only shallow realisations is more complex change, and it turns out we weren't even testing for the benefits that derivation hashes (modulo FODs) provided in the deep realisation case, so I now just want to do this first. Doing this gets the binary cache data structures in order, which will unblock the Hydra fixed-output-derivation tracking work. I don't want to delay that work while I figure out the changes needed for shallow-realisations only. This reverts commit bab1cda0e6c30e25460b5a9c809589d3948f35df. --- src/libcmd/built-path.cc | 18 +- src/libexpr/primops.cc | 23 +- src/libstore-tests/common-protocol.cc | 55 ----- .../data/realisation/simple.json | 12 +- .../data/realisation/with-signature.json | 16 +- .../data/serve-protocol/build-result-2.8.bin | Bin 0 -> 360 bytes .../data/serve-protocol/drv-output-2.8.bin | Bin 0 -> 160 bytes .../data/serve-protocol/realisation-2.8.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-2.8.bin | Bin 0 -> 96 bytes .../worker-protocol/build-result-1.39.bin | Bin 0 -> 424 bytes .../data/worker-protocol/drv-output-1.39.bin | Bin 0 -> 160 bytes .../data/worker-protocol/realisation-1.39.bin | Bin 0 -> 176 bytes .../unkeyed-realisation-1.39.bin | Bin 0 -> 96 bytes src/libstore-tests/dummy-store.cc | 11 +- src/libstore-tests/realisation.cc | 65 ++--- src/libstore-tests/serve-protocol.cc | 140 ++++++----- src/libstore-tests/worker-protocol.cc | 173 +++++++------ src/libstore/binary-cache-store.cc | 12 +- .../build/derivation-building-goal.cc | 7 +- src/libstore/build/derivation-goal.cc | 47 +--- .../build/drv-output-substitution-goal.cc | 7 +- src/libstore/common-protocol.cc | 28 --- src/libstore/daemon.cc | 26 +- src/libstore/derivations.cc | 231 +++++++++++------- src/libstore/dummy-store.cc | 4 +- .../include/nix/store/binary-cache-store.hh | 9 +- .../store/build/derivation-building-misc.hh | 1 - .../nix/store/build/derivation-goal.hh | 2 - .../include/nix/store/common-protocol-impl.hh | 5 +- .../include/nix/store/common-protocol.hh | 7 +- src/libstore/include/nix/store/derivations.hh | 61 +++-- .../include/nix/store/dummy-store-impl.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 +- src/libstore/include/nix/store/realisation.hh | 74 +++--- .../include/nix/store/serve-protocol-impl.hh | 6 +- .../include/nix/store/serve-protocol.hh | 15 +- .../include/nix/store/worker-protocol-impl.hh | 6 +- .../include/nix/store/worker-protocol.hh | 17 +- src/libstore/local-binary-cache-store.cc | 5 +- src/libstore/local-store.cc | 8 +- src/libstore/misc.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 49 ++-- src/libstore/realisation.cc | 85 ++++--- src/libstore/remote-store.cc | 44 +--- src/libstore/restricted-store.cc | 15 +- src/libstore/serve-protocol.cc | 109 ++++++++- src/libstore/store-api.cc | 7 +- src/libstore/unix/build/derivation-builder.cc | 5 +- src/libstore/worker-protocol.cc | 144 ++++++++++- src/nix/build-remote/build-remote.cc | 6 +- src/nix/develop.cc | 10 +- src/perl/lib/Nix/Store.xs | 7 +- tests/functional/ca/substitute.sh | 6 +- 53 files changed, 909 insertions(+), 700 deletions(-) create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-1.39.bin create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f18493..f60e4499c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; + DrvOutput key{ + .drvPath = p.drvPath->outPath(), + .outputName = outputName, + }; auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); + // We’ve built it, so we must have the realisation. + assert(thisRealisation); + res.insert(Realisation{*thisRealisation, key}); } else { res.insert(outputPath); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d1aae64fa..fb458d0fc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1718,28 +1718,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); } - auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); - switch (hashModulo.kind) { - case DrvHash::Kind::Regular: - for (auto & i : outputs) { - auto h = get(hashModulo.hashes, i); - if (!h) - state.error("derivation produced no hash for output '%s'", i).atPos(v).debugThrow(); - auto outPath = state.store->makeOutputPath(i, *h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign( - i, - DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }); - } - break; - ; - case DrvHash::Kind::Deferred: - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{}); - } - } + resolveInputAddressed(*state.store, drv); } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 6bd7beb44..71f42517f 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -108,61 +108,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e721..1e4760b56 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index a28848cb0..4a73ed3ef 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,12 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdfasdfasdf" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdfasdfasdf" + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa072599578a351e0f289c1c6bbdc5c1ff90107c GIT binary patch literal 360 zcmZQ&fBjf;Y*hsmSsV}eS+ z%uh-z0*mN_Nd^Y}yvz#y;*$KLRQ+_ra`TKz<3e1tE=(^-E6lvK{Cp6XfgJN-dO-#N E0Qb5q`2YX_ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.39.bin b/src/libstore-tests/data/worker-protocol/build-result-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..11bec3e6e3b7a82f7b2d792d687a807644ff9d05 GIT binary patch literal 424 zcmZQ&fBO)Le|4^xkB4qQI< ztSm?kobS(|0^&1)2nGfQn0t~Ei@+lKV3L7BKQFUFzqlm7C{;h*u-rVO(zp;8tqao& U(h4&#Ek7T`Wgy2qm|lN>LeDBQuy}U`R@=0<$PJj|FTB M14ChHX$6Q00QuS{z5oCK literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..5295ee34400a58e06c919eaf4b0c70b95b713fc9 GIT binary patch literal 176 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQuy}U`R@=0<-kNBm)D9 Z<}olq^|3(d#Nw1R5EI5PEKe;0@c}{FC(r-@ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-1.39.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f4ebcb2eb1622e1c56b462f36a7db1e863b345 GIT binary patch literal 96 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6GR&W3zSYQPDukXVf@1K)FKcc E0Dd79Jpcdz literal 0 HcmV?d00001 diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 36a0c3019..f53fc03b4 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, make_ref(value)}}}); + store->buildTrace.insert({drvPath, {{outputName, make_ref(value)}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 66f4707e9..3087bd1b5 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,45 +44,30 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, -}; - -INSTANTIATE_TEST_SUITE_P( - RealisationJSON, - RealisationJsonTest, - ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }(), - })); - -/** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. - */ -TEST_F(RealisationTest, dependent_realisations_from_json) -{ - readJsonTest("with-dependent-realisations", simple); -} +INSTANTIATE_TEST_SUITE_P(RealisationJSON, RealisationJsonTest, ([] { + Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", + }, + }; + return ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }(), + }); + }())); } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index afc35d03c..1bffc08ef 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -73,16 +73,16 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + 2 << 8 | 8, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -91,46 +91,27 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + 2 << 8 | 8, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + 2 << 8 | 8, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -180,7 +161,10 @@ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2 t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -206,27 +190,65 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, buildResult_2_8, "build-result-2.8", 2 << 8 | 8, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index a1f68e231..853568940 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -129,61 +129,42 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-1.39", + 1 << 8 | 39, (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_1_39, + "unkeyed-realisation-1.39", + 1 << 8 | 39, + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_1_39, + "realisation-1.39", + 1 << 8 | 39, + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = {"asdf", "qwer"}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -205,7 +186,10 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully right + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -224,25 +208,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -251,7 +223,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -277,27 +250,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -310,7 +269,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ using namespace std::literals::chrono_literals; std::tuple t{ @@ -336,27 +296,60 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult_1_39, "build-result-1.39", 1 << 8 | 39, ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, + .errorMsg = "no idea why", + }}}, + BuildResult{ + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 274e47271..6a213da57 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,7 +535,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -551,7 +554,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index c72130142..8e06b2e08 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -212,9 +212,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild() given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1106,7 +1105,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8e9ff5e4..d27485abe 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -36,12 +36,6 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} , buildMode(buildMode) { @@ -100,7 +94,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (settings.useSubstitutes && drvOptions.substitutesAllowed()) { if (!checkResult) - waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput}))); + waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{drvPath, wantedOutput}))); else { auto * cap = getDerivationCA(*drv); waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( @@ -167,12 +161,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -181,7 +170,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -189,7 +178,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -205,7 +194,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvHash = *outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -251,16 +240,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -385,7 +365,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -425,7 +405,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -448,16 +428,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke { buildResult.inner = BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }; mcExpectedBuilds.reset(); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 58f3de2b7..34a263edd 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -12,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -107,7 +107,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -122,7 +123,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index b069c9498..6b819fd6e 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -46,34 +46,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 937946134..0b494d0d7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -963,33 +963,31 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { - auto outputId = DrvOutput::parse(readString(conn.from)); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + // TODO move to WorkerProto::Serialise and friends + // if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // throw Error("old-style build traces no longer supported"); + //} + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(conn.from)); - auto info = store->queryRealisation(outputId); + auto outputId = WorkerProto::Serialise::read(*store, rconn); + std::optional info = *store->queryRealisation(outputId); logger->stopWork(); if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); + } else if (GET_PROTOCOL_MINOR(conn.protoVersion) < 39) { + // No longer support this format + WorkerProto::write(*store, wconn, StringSet{}); } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); + WorkerProto::write(*store, wconn, info); } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index e6ac08fd9..b9ec4d64b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -866,13 +866,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -895,12 +896,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -910,54 +909,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; + } + const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; } } - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); - } - - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1107,22 +1118,39 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +void resolveInputAddressed(Store & store, Derivation & drv) { - drv.applyRewrites(rewrites); + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drv.name); + drv.env.insert_or_assign(outputName, store.printStorePath(outPath)); + output = DerivationOutput::InputAddressed{ + .path = std::move(outPath), + }; + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + deferred (waiting to be input-addressed). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + // Nothing to do, already deferred + }, + }, + hashModulo().raw); } } } @@ -1205,9 +1233,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); - return resolved; + Derivation resolved2{std::move(resolved)}; + + resolveInputAddressed(store, resolved2); + + return resolved2; } void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const @@ -1235,46 +1267,79 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const // combinations that are currently prohibited. type(); - std::optional hashesModulo; - for (auto & i : outputs) { + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, *this, true); + } + return *hashModulo_; + }; + + for (auto & [outputName, output] : outputs) { std::visit( overloaded{ [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, *this, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error( - "derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - i.first); - StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error( - "derivation '%s' has incorrect output '%s', should be '%s'", - store.printStorePath(drvPath), - store.printStorePath(doia.path), - store.printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + StorePath recomputed = store.makeOutputPath(outputName, drvHash, drvName); + if (doia.path != recomputed) + throw Error( + "derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), + store.printStorePath(doia.path), + store.printStorePath(recomputed)); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + throw Error( + "derivation '%s' has output '%s', but derivation is not yet ready to be input-addressed", + store.printStorePath(drvPath), + store.printStorePath(doia.path)); + }, + }, + hashModulo().raw); + envHasRightPath(doia.path, outputName); }, [&](const DerivationOutput::CAFixed & dof) { - auto path = dof.path(store, drvName, i.first); - envHasRightPath(path, i.first); + auto path = dof.path(store, drvName, outputName); + envHasRightPath(path, outputName); }, [&](const DerivationOutput::CAFloating &) { /* Nothing to check */ }, [&](const DerivationOutput::Deferred &) { /* Nothing to check */ + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + throw Error( + "derivation '%s' has deferred output '%s', yet is ready to be input-addressed", + store.printStorePath(drvPath), + outputName); + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed. */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + /* Nothing to check */ + }, + }, + hashModulo().raw); }, [&](const DerivationOutput::Impure &) { /* Nothing to check */ }, }, - i.second.raw); + output.raw); } } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 218a4e2ed..2a4c21b5f 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -332,7 +332,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { auto ref = make_ref(output); - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, ref}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, ref}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, make_ref(output)); }); } @@ -341,7 +341,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(it->second.get_ptr()); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index e64dc3eae..8623d5f4e 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -82,8 +82,13 @@ protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895), and this serves as some light + * versioning. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -92,7 +97,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa178..8d6892839 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 0fe610987..c732fd441 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -71,8 +71,6 @@ private: */ std::unique_ptr drv; - const Hash outputHash; - const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c..d0eebe0bc 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index c1d22fa6c..45bec3ff2 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -12,7 +12,6 @@ struct Source; class StorePath; struct ContentAddress; struct DrvOutput; -struct Realisation; /** * Shared serializers between the worker protocol, serve protocol, and a @@ -70,8 +69,6 @@ template<> DECLARE_COMMON_SERIALISER(ContentAddress); template<> DECLARE_COMMON_SERIALISER(DrvOutput); -template<> -DECLARE_COMMON_SERIALISER(Realisation); #define COMMA_ , template @@ -81,8 +78,8 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); #undef COMMA_ /** diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 259314d3f..045ba0eab 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -427,34 +427,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, - - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + using DrvHash = Hash; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. */ - Kind kind; -}; + using CaOutputHashes = std::map; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + /** + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. + */ + using DeferredDrv = std::monostate; + + using Raw = std::variant; + + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -480,15 +485,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -503,7 +510,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 71869cb7e..42fa32d61 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -49,7 +49,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map>> buildTrace; + boost::concurrent_flat_map>> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340..e1a80e8dc 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 3691c14fb..e5ac6d73d 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,33 +18,40 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Opaque`, but the derivation is - * identified by its "hash modulo" instead of by its store path. + * This is similar to a `DerivedPath::Built`, except it is only a single + * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. */ struct DrvOutput { /** - * The hash modulo of the derivation. - * - * Computed from the derivation itself for most types of - * derivations, but computed from the (fixed) content address of the - * output for fixed-output derivations. + * The store path to the derivation */ - Hash drvHash; + StorePath drvPath; /** * The name of the output. */ OutputName outputName; + /** + * Skips the store dir on the `drvPath` + */ std::string to_string() const; - std::string strHash() const - { - return drvHash.to_string(HashFormat::Base16, true); - } + /** + * Skips the store dir on the `drvPath` + */ + static DrvOutput from_string(std::string_view); - static DrvOutput parse(const std::string &); + /** + * Includes the store dir on `drvPath` + */ + std::string render(const StoreDirConfig & store) const; + + /** + * Includes the store dir on `drvPath` + */ + static DrvOutput parse(const StoreDirConfig & store, std::string_view); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -64,6 +71,16 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; + /** + * Just check the `outPath`. Signatures don't matter for this. + * Callers must ensure that the corresponding key is the same for + * most use-cases. + */ + bool isCompatibleWith(const UnkeyedRealisation & other) const + { + return outPath == other.outPath; + } + const StorePath & getPath() const { return outPath; @@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; - bool isCompatibleWith(const UnkeyedRealisation & other) const; - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; - -/** - * Collection type for multiple derivations' outputs' `Realisation`s. - * - * `DrvOutput` is used because in general the derivations are not all - * the same, so we need to identify firstly which derivation, and - * secondly which output of that derivation. - */ -typedef std::map DrvOutputs; +typedef std::map SingleDrvOutputs; struct OpaquePath { @@ -149,19 +155,17 @@ struct RealisedPath class MissingRealisation : public Error { public: - MissingRealisation(DrvOutput & outputId) - : MissingRealisation(outputId.outputName, outputId.strHash()) + MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) + : MissingRealisation(store, outputId.drvPath, outputId.outputName) { } - MissingRealisation(std::string_view drv, OutputName outputName) - : Error( - "cannot operate on output '%s' of the " - "unbuilt derivation '%s'", - outputName, - drv) - { - } + MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); + MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName); }; } // namespace nix diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index a9617165a..24fc3c9ab 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 974bf42d5..1b548b517 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) +#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,6 +18,9 @@ struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -178,6 +181,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> +DECLARE_SERVE_SERIALISER(DrvOutput); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_SERVE_SERIALISER(Realisation); +template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -190,8 +199,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44..605663d2e 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 6ae5fdcbc..4374ba7a6 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,7 +12,7 @@ namespace nix { /* Note: you generally shouldn't change the protocol version. Define a new `WorkerProto::Feature` instead. */ -#define PROTOCOL_VERSION (1 << 8 | 38) +#define PROTOCOL_VERSION (1 << 8 | 39) #define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -35,6 +35,9 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; @@ -261,6 +264,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(DrvOutput); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_WORKER_SERIALISER(Realisation); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(std::optional); @@ -277,8 +288,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 63730a01b..641a397b3 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -56,9 +56,10 @@ protected: void upsertFile( const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override { - auto path2 = config->binaryCacheDir + "/" + path; + auto path2 = std::filesystem::path{config->binaryCacheDir} / path; static std::atomic counter{0}; - Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); + createDirs(path2.parent_path()); + auto tmp = path2 + fmt(".tmp.%d.%d", getpid(), ++counter); AutoDelete del(tmp, false); writeFile(tmp, source); std::filesystem::rename(tmp, path2); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 98f2d70af..cae2dc4a0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -110,8 +110,6 @@ struct LocalStore::State::Stmts SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; - SQLiteStmt QueryRealisationReferences; - SQLiteStmt AddRealisationReference; }; LocalStore::LocalStore(ref config) @@ -621,7 +619,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", combinedSignatures))(info.id.strHash())(info.id.outputName) + .use()(concatStringsSep(" ", combinedSignatures))(info.id.drvPath.to_string())(info.id.outputName) .exec(); } else { throw Error( @@ -635,7 +633,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", info.signatures)) .exec(); } @@ -1553,7 +1551,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si std::optional> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6829563c2..b46d900b7 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -240,16 +240,15 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto [outputName, hash] : outputHashes) { + for (auto & [outputName, _] : drv->outputs) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({hash, outputName}); + auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) continue; found = true; @@ -362,7 +361,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -386,7 +385,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 11608a667..116957e55 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -44,10 +44,16 @@ create table if not exists NARs ( create table if not exists Realisations ( cache integer not null, - outputId text not null, - content blob, -- Json serialisation of the realisation, or null if the realisation is absent + + drvPath text not null, + outputName text not null, + + -- The following are null if the realisation is absent + outputPath text, + sigs text, + timestamp integer not null, - primary key (cache, outputId), + primary key (cache, drvPath, outputName), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -121,24 +127,24 @@ public: state->insertRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp) - values (?, ?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, outputPath, sigs, timestamp) + values (?, ?, ?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp) - values (?, ?, ?) + insert or replace into Realisations(cache, drvPath, outputName, timestamp) + values (?, ?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select content from Realisations - where cache = ? and outputId = ? and - ((content is null and timestamp > ?) or - (content is not null and timestamp > ?)) + select outputPath, sigs from Realisations + where cache = ? and drvPath = ? and outputName = ? and + ((outputPath is null and timestamp > ?) or + (outputPath is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -295,22 +301,27 @@ public: auto now = time(0); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache)); if (!queryRealisation.next()) - return {oUnknown, 0}; + return {oUnknown, nullptr}; if (queryRealisation.isNull(0)) - return {oInvalid, 0}; + return {oInvalid, nullptr}; try { return { oValid, - std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + std::make_shared( + UnkeyedRealisation{ + .outPath = StorePath{queryRealisation.getStr(0)}, + .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), + }, + id), }; } catch (Error & e) { - e.addTrace({}, "while parsing the local disk cache"); + e.addTrace({}, "reading build trace key-value from the local disk cache"); throw; } }); @@ -355,7 +366,9 @@ public: auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) + .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( + realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( + time(0)) .exec(); }); } @@ -366,7 +379,7 @@ public: auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 2075b39e6..aaad8f662 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,28 +8,34 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const std::string & strRep) +DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) { - size_t n = strRep.find("!"); - if (n == strRep.npos) - throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); - + size_t n = s.rfind('^'); + if (n == s.npos) + throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); return DrvOutput{ - .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), - .outputName = strRep.substr(n + 1), + .drvPath = store.parseStorePath(s.substr(0, n)), + .outputName = OutputName{s.substr(n + 1)}, }; } +std::string DrvOutput::render(const StoreDirConfig & store) const +{ + return std::string(store.printStorePath(drvPath)) + "^" + outputName; +} + std::string DrvOutput::to_string() const { - return strHash() + "!" + outputName; + return std::string(drvPath.to_string()) + "^" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - nlohmann::json serialized = Realisation{*this, key}; - serialized.erase("signatures"); - return serialized.dump(); + auto serialised = static_cast(Realisation{*this, key}); + auto value = serialised.find("value"); + assert(value != serialised.end()); + value->erase("signatures"); + return serialised.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -61,9 +67,20 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) + : Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) { - return outPath == other.outPath; +} + +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName) + : MissingRealisation{store, drvPathResolved, outputName} +{ + addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); } } // namespace nix @@ -74,24 +91,32 @@ using namespace nix; DrvOutput adl_serializer::from_json(const json & json) { - return DrvOutput::parse(getString(json)); + auto obj = getObject(json); + + return { + .drvPath = valueAt(obj, "drvPath"), + .outputName = getString(valueAt(obj, "outputName")), + }; } void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) { - json = drvOutput.to_string(); + json = { + {"drvPath", drvOutput.drvPath}, + {"outputName", drvOutput.outputName}, + }; } -UnkeyedRealisation adl_serializer::from_json(const json & json0) +UnkeyedRealisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); StringSet signatures; - if (auto signaturesOpt = optionalValueAt(json, "signatures")) - signatures = *signaturesOpt; + if (auto * signaturesJson = get(obj, "signatures")) + signatures = getStringSet(*signaturesJson); - return UnkeyedRealisation{ - .outPath = valueAt(json, "outPath"), + return { + .outPath = valueAt(obj, "outPath"), .signatures = signatures, }; } @@ -101,25 +126,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - // back-compat - {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json0) +Realisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); - return Realisation{ - static_cast(json0), - valueAt(json, "id"), + return { + static_cast(valueAt(obj, "value")), + static_cast(valueAt(obj, "key")), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = static_cast(r); - json["id"] = r.id; + json = { + {"key", r.id}, + {"value", static_cast(r)}, + }; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 949a51f18..6a8e48a6d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -507,7 +507,7 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 39) { warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -516,21 +516,12 @@ void RemoteStore::queryRealisationUncached( conn->to << id.to_string(); conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { - auto outPaths = WorkerProto::Serialise>::read(*this, *conn); - if (outPaths.empty()) - return nullptr; - return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); - } else { - auto realisations = WorkerProto::Serialise>::read(*this, *conn); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); - - callback(std::shared_ptr(real)); + callback([&]() -> std::shared_ptr { + auto realisation = WorkerProto::Serialise>::read(*this, *conn); + if (!realisation) + return nullptr; + return std::make_shared(*realisation); + }()); } catch (...) { return callback.rethrow(); } @@ -612,30 +603,19 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto drv = evalStore->readDerivation(drvPath); - const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - printStorePath(drvPath), - output); - auto outputId = DrvOutput{*outputHash, output}; + auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(outputId); - success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); + throw MissingRealisation(*this, outputId); + success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( output, - Realisation{ - UnkeyedRealisation{ - .outPath = outputPath, - }, - outputId, + UnkeyedRealisation{ + .outPath = outputPath, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index b4d696a53..99650870e 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -281,9 +281,18 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * pathBuilt = std::get_if(&result.path)) { + // TODO ugly extra IO + auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert( + {output, + { + .drvPath = drvPath, + .outputName = outputName, + }}); + } } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 51b575fcd..d5f8a9ea9 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include @@ -24,10 +25,19 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + success.builtOutputs = ServeProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -51,15 +61,18 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 8) { + ServeProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + // We no longer support these types of realisations + ServeProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -144,4 +157,82 @@ void ServeProto::Serialise::write( } } +UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = ServeProto::Serialise::read(store, conn); + auto signatures = ServeProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void ServeProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.outPath); + ServeProto::write(store, conn, info.signatures); +} + +DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = ServeProto::Serialise::read(store, conn); + auto outputName = ServeProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 8) { + throw Error( + "daemon protocol %d.%d is too old (< 2.8) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + ServeProto::write(store, conn, info.drvPath); + ServeProto::write(store, conn, info.outputName); +} + +Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = ServeProto::Serialise::read(store, conn); + auto unkeyed = ServeProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + ServeProto::write(store, conn, info.id); + ServeProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c70af9301..71642431f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -367,9 +367,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); + for (auto & [outputName, _] : drv.outputs) { + auto realisation = queryRealisation(DrvOutput{path, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -389,7 +388,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(printStorePath(path), outName); + throw MissingRealisation(*this, path, outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index c2ef730dc..35b0063db 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1870,7 +1870,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{oldinfo->outputHash, outputName}, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a17d2c028..c6719974b 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include #include @@ -132,7 +133,7 @@ void WorkerProto::Serialise::write( throw Error( "trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MAJOR(conn.version) >> 8, GET_PROTOCOL_MINOR(conn.version)); }, [&](std::monostate) { @@ -174,14 +175,24 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + success.builtOutputs = WorkerProto::Serialise>::read(store, conn); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } if (BuildResult::Success::statusIs(rawStatus)) { @@ -205,20 +216,24 @@ void WorkerProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputsFullKey); + + if (GET_PROTOCOL_MINOR(conn.version) >= 39) { + WorkerProto::write(store, conn, builtOutputs); + } else if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + // Don't support those types of realisations anymore. + WorkerProto::write(store, conn, StringMap{}); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -309,4 +324,113 @@ void WorkerProto::Serialise::write( } } +UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto outPath = WorkerProto::Serialise::read(store, conn); + auto signatures = WorkerProto::Serialise::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.39) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.outPath); + WorkerProto::write(store, conn, info.signatures); +} + +std::optional +WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + // Hack to improve compat + (void) WorkerProto::Serialise::read(store, conn); + return std::nullopt; + } else { + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::Serialise::read(store, conn); + default: + throw Error("Invalid optional build trace from remote"); + } + } +} + +void WorkerProto::Serialise>::write( + const StoreDirConfig & store, WriteConn conn, const std::optional & info) +{ + if (!info) { + conn.to << uint8_t{0}; + } else { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *info); + } +} + +DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + + auto drvPath = WorkerProto::Serialise::read(store, conn); + auto outputName = WorkerProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) < 39) { + throw Error( + "daemon protocol %d.%d is too old (< 1.29) to understand build trace", + GET_PROTOCOL_MAJOR(conn.version) >> 8, + GET_PROTOCOL_MINOR(conn.version)); + } + WorkerProto::write(store, conn, info.drvPath); + WorkerProto::write(store, conn, info.outputName); +} + +Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = WorkerProto::Serialise::read(store, conn); + auto unkeyed = WorkerProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + WorkerProto::write(store, conn, info.id); + WorkerProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index ffb77ddf1..1e52daadc 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } - auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputHash = outputHashes.at(outputName); - auto thisOutputId = DrvOutput{thisOutputHash, outputName}; + auto thisOutputId = DrvOutput{*drvPath, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); + missingRealisations.insert({newRealisation, thisOutputId}); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d23dce10b..0f894b9de 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -279,16 +279,8 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore output.second = DerivationOutput::Deferred{}; drv.env[output.first] = ""; } - auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); - for (auto & output : drv.outputs) { - Hash h = hashesModulo.hashes.at(output.first); - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = DerivationOutput::InputAddressed{ - .path = outPath, - }; - drv.env[output.first] = store->printStorePath(outPath); - } + resolveInputAddressed(*evalStore, drv); } auto shellDrvPath = writeDerivation(*evalStore, drv); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 93e9f0f95..6d5dcb400 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * outputId) +StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput{ + .drvPath = THIS->store->parseStorePath(drvPath), + .outputName = outputName, + }); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 2f6ebcef5..c4cbf2c96 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/realisations" +rm -r "$REMOTE_STORE_DIR/build-trace" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm "$REMOTE_STORE_DIR"/realisations/* +rm -r "$REMOTE_STORE_DIR"/build-trace/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From c893454926bf1dc96848fa0e48d6625f11fafdb7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Oct 2025 13:49:54 -0400 Subject: [PATCH 155/155] Fixup issues with bytes --- src/libstore-tests/dummy-store.cc | 3 ++- src/libstore/dummy-store.cc | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index f53fc03b4..3ac3c2651 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -1,6 +1,7 @@ #include #include +#include "nix/util/bytes.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/globals.hh" @@ -105,7 +106,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { auto sc = make_ref(); sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ .executable = false, - .contents = "asdf", + .contents = to_owned(as_bytes("asdf")), }}; return sc; }(), diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 2a4c21b5f..b4b1d1f61 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,5 +1,6 @@ #include "nix/store/store-registration.hh" #include "nix/util/archive.hh" +#include "nix/util/bytes.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/json-utils.hh" @@ -160,7 +161,7 @@ struct DummyStoreImpl : DummyStore .method = ContentAddressMethod::Raw::Text, .hash = hashString( HashAlgorithm::SHA256, - std::get(accessor->root->raw).contents), + to_str(std::get(accessor->root->raw).contents)), }; callback(std::move(info)); return; @@ -428,7 +429,7 @@ ref adl_serializer>::from_json(const json & json) auto vref = make_ref(v2); res->buildTrace.insert_or_visit( { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), + StorePath{k0}, {{k1, vref}}, }, [&](auto & kv) { kv.second.insert_or_assign(k1, vref); }); @@ -464,7 +465,7 @@ void adl_serializer>::to_json(json & json, const ref auto obj = json::object(); val->buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = *v2; });