From 01388b3e78582553cc30ee772db5b5aba8d89edd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Jun 2025 12:09:43 +0200 Subject: [PATCH 01/20] Give unit tests access to a $HOME directory Also, don't try to access cache.nixos.org in the libstore unit tests. --- src/libflake-tests/meson.build | 1 + src/libflake-tests/package.nix | 20 ++++++++------------ src/libstore-tests/meson.build | 1 + src/libstore-tests/nix_api_store.cc | 16 +--------------- src/libstore-tests/package.nix | 18 +++++++----------- 5 files changed, 18 insertions(+), 38 deletions(-) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 593b0e18d..8c082c7e0 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -60,6 +60,7 @@ test( env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', 'NIX_CONFIG': 'extra-experimental-features = flakes', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 714f3791a..1a8afd6ea 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -56,18 +56,14 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - export NIX_CONFIG="extra-experimental-features = flakes" - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export NIX_CONFIG="extra-experimental-features = flakes" + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 8a1ff40f0..8b9893b23 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -100,6 +100,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 3d9f7908b..05373cb88 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -28,10 +28,6 @@ TEST_F(nix_api_store_test, nix_store_get_uri) TEST_F(nix_api_util_context, nix_store_get_storedir_default) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles - GTEST_SKIP(); - } nix_libstore_init(ctx); Store * store = nix_store_open(ctx, nullptr, nullptr); assert_ctx_ok(); @@ -141,10 +137,6 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // Can't open default store from within sandbox - GTEST_SKIP(); - } auto tmp = nix::createTempDir(); std::string storeRoot = tmp + "/store"; std::string stateDir = tmp + "/state"; @@ -184,13 +176,7 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // TODO: override NIX_CACHE_HOME? - // skipping test in sandbox because narinfo cache can't be written - GTEST_SKIP(); - } - - Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + Store * store = nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir()).c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index b39ee7fa7..1f3701c7f 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -73,17 +73,13 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; From f29acd5bbc0505ea6a78a51e4828a22630dd0ff1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Jul 2025 11:12:55 +0200 Subject: [PATCH 02/20] Use writableTmpDirAsHomeHook --- src/libflake-tests/package.nix | 4 ++-- src/libstore-tests/package.nix | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 1a8afd6ea..397ef4192 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -3,6 +3,7 @@ buildPackages, stdenv, mkMesonExecutable, + writableTmpDirAsHomeHook, nix-flake, nix-flake-c, @@ -55,12 +56,11 @@ mkMesonExecutable (finalAttrs: { runCommand "${finalAttrs.pname}-run" { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + buildInputs = [ writableTmpDirAsHomeHook ]; } ('' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} export NIX_CONFIG="extra-experimental-features = flakes" - export HOME="$TMPDIR/home" - mkdir -p "$HOME" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 1f3701c7f..62c7e136b 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -3,6 +3,7 @@ buildPackages, stdenv, mkMesonExecutable, + writableTmpDirAsHomeHook, nix-store, nix-store-c, @@ -72,11 +73,10 @@ mkMesonExecutable (finalAttrs: { runCommand "${finalAttrs.pname}-run" { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + buildInputs = [ writableTmpDirAsHomeHook ]; } ('' export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} - export HOME="$TMPDIR/home" - mkdir -p "$HOME" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); From 778156072444fdd06812b89e86169bf45f2425df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Jul 2025 11:26:06 +0200 Subject: [PATCH 03/20] libstore-tests: Fix impurity trying to access the Nix daemon This failed on macOS: nix-store-tests-run> C++ exception with description "../nix_api_store.cc:33: nix_err_code(ctx) != NIX_OK, message: error: getting status of '/nix/var/nix/daemon-socket/socket': Operation not permitted" thrown in the test body. --- src/libstore-tests/meson.build | 1 + src/libstore-tests/package.nix | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 8b9893b23..79f21620e 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -101,6 +101,7 @@ test( env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', 'HOME': meson.current_build_dir() / 'test-home', + 'NIX_REMOTE': meson.current_build_dir() / 'test-home' / 'store', }, protocol : 'gtest', ) diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 62c7e136b..f606604ba 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -77,6 +77,7 @@ mkMesonExecutable (finalAttrs: { } ('' export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} + export NIX_REMOTE=$HOME/store ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); From 74a144ce9831c65371f7482fc6ae748872df679d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2025 11:53:36 +0200 Subject: [PATCH 04/20] Require Boost 1.81.0 or higher Note: this version of Boost was released in December 2022. --- src/libexpr/include/nix/expr/symbol-table.hh | 14 +------------- src/libexpr/meson.build | 1 + src/libutil/serialise.cc | 4 ---- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index 20a05a09d..1249bdb88 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -7,12 +7,7 @@ #include "nix/util/error.hh" #include -#define USE_FLAT_SYMBOL_SET (BOOST_VERSION >= 108100) -#if USE_FLAT_SYMBOL_SET -# include -#else -# include -#endif +#include namespace nix { @@ -214,12 +209,7 @@ private: * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store. * ChunkedVector references are never invalidated. */ -#if USE_FLAT_SYMBOL_SET boost::unordered_flat_set symbols{SymbolStr::chunkSize}; -#else - using SymbolValueAlloc = std::pmr::polymorphic_allocator; - boost::unordered_set symbols{SymbolStr::chunkSize, {&buffer}}; -#endif public: @@ -287,5 +277,3 @@ struct std::hash return std::hash{}(s.id); } }; - -#undef USE_FLAT_SYMBOL_SET diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index f5adafae0..533030359 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -41,6 +41,7 @@ boost = dependency( 'boost', modules : ['container', 'context'], include_type: 'system', + version: '>=1.81.0' ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 55397c6d4..a74531582 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -194,10 +194,6 @@ size_t StringSource::read(char * data, size_t len) } -#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 -#error Coroutines are broken in this version of Boost! -#endif - std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink From ca9f2028b020447559d509252c31a145f92dfaff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2025 12:27:17 +0200 Subject: [PATCH 05/20] Simplify SymbolTable::create() --- src/libexpr/include/nix/expr/symbol-table.hh | 14 +------------- src/libexpr/meson.build | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index 1249bdb88..4dedf3d91 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -220,19 +220,7 @@ public: // Most symbols are looked up more than once, so we trade off insertion performance // for lookup performance. // FIXME: make this thread-safe. - return [&](T && key) -> Symbol { - if constexpr (requires { symbols.insert(key); }) { - auto [it, _] = symbols.insert(key); - return Symbol(*it); - } else { - auto it = symbols.find(key); - if (it != symbols.end()) - return Symbol(*it); - - it = symbols.emplace(key).first; - return Symbol(*it); - } - }(SymbolStr::Key{store, s, stringAlloc}); + return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first); } std::vector resolve(const std::vector & symbols) const diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 533030359..43e4b9c98 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -41,7 +41,7 @@ boost = dependency( 'boost', modules : ['container', 'context'], include_type: 'system', - version: '>=1.81.0' + version: '>=1.82.0' ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. From 3e9a100bdf64664bea84aa9eb8ae620945d2651b Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Sun, 13 Jul 2025 22:49:06 +0200 Subject: [PATCH 06/20] docker: set default parameters values --- docker.nix | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docker.nix b/docker.nix index f59492025..410e4a178 100644 --- a/docker.nix +++ b/docker.nix @@ -1,10 +1,10 @@ { # Core dependencies - pkgs, - lib, - dockerTools, - runCommand, - buildPackages, + pkgs ? import { }, + lib ? pkgs.lib, + dockerTools ? pkgs.dockerTools, + runCommand ? pkgs.runCommand, + buildPackages ? pkgs.buildPackages, # Image configuration name ? "nix", tag ? "latest", @@ -28,24 +28,24 @@ }, Cmd ? [ (lib.getExe bashInteractive) ], # Default Packages - nix, - bashInteractive, - coreutils-full, - gnutar, - gzip, - gnugrep, - which, - curl, - less, - wget, - man, - cacert, - findutils, - iana-etc, - gitMinimal, - openssh, + nix ? pkgs.nix, + bashInteractive ? pkgs.bashInteractive, + coreutils-full ? pkgs.coreutils-full, + gnutar ? pkgs.gnutar, + gzip ? pkgs.gzip, + gnugrep ? pkgs.gnugrep, + which ? pkgs.which, + curl ? pkgs.curl, + less ? pkgs.less, + wget ? pkgs.wget, + man ? pkgs.man, + cacert ? pkgs.cacert, + findutils ? pkgs.findutils, + iana-etc ? pkgs.iana-etc, + gitMinimal ? pkgs.gitMinimal, + openssh ? pkgs.openssh, # Other dependencies - shadow, + shadow ? pkgs.shadow, }: let defaultPkgs = [ From 04f6974d2c47ae3cc44733adb707107a675e2c92 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 13 Jul 2025 15:21:01 +0300 Subject: [PATCH 07/20] ci: Dogfood Nix from master --- .../actions/install-nix-action/action.yaml | 50 +++++++++++++++++++ .github/workflows/ci.yml | 14 ++++-- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 .github/actions/install-nix-action/action.yaml diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml new file mode 100644 index 000000000..28103f589 --- /dev/null +++ b/.github/actions/install-nix-action/action.yaml @@ -0,0 +1,50 @@ +name: "Install Nix" +description: "Helper action for installing Nix with support for dogfooding from master" +inputs: + dogfood: + description: "Whether to use Nix installed from the latest artifact from master branch" + required: true # Be explicit about the fact that we are using unreleased artifacts + extra_nix_config: + description: "Gets appended to `/etc/nix/nix.conf` if passed." + install_url: + description: "URL of the Nix installer" + required: false + default: "https://releases.nixos.org/nix/nix-2.30.1/install" + github_token: + description: "Github token" + required: true +runs: + using: "composite" + steps: + - name: "Download nix install artifact from master" + shell: bash + id: download-nix-installer + if: ${{ inputs.dogfood }} + run: | + RUN_ID=$(gh run list --repo "$DOGFOOD_REPO" --workflow ci.yml --branch master --status success --json databaseId --jq ".[0].databaseId") + + if [ "$RUNNER_OS" == "Linux" ]; then + INSTALLER_ARTIFACT="installer-linux" + elif [ "$RUNNER_OS" == "macOS" ]; then + INSTALLER_ARTIFACT="installer-darwin" + else + echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" + exit 1 + fi + + INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$INSTALLER_ARTIFACT" + mkdir -p "$INSTALLER_DOWNLOAD_DIR" + + gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" + echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" + + echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" + env: + GH_TOKEN: ${{ inputs.github_token }} + DOGFOOD_REPO: "NixOS/nix" + - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 + with: + # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 + install_url: ${{ inputs.dogfood && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} + install_options: ${{ inputs.dogfood && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} + extra_nix_config: ${{ inputs.extra_nix_config }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac749bc3f..2531ee020 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,13 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v31 + - uses: ./.github/actions/install-nix-action with: - install_url: "https://releases.nixos.org/nix/nix-2.29.1/install" - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + dogfood: true + extra_nix_config: + experimental-features = nix-command flakes + github_token: ${{ secrets.GITHUB_TOKEN }} + - run: nix flake show --all-systems --json tests: strategy: @@ -36,9 +39,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v31 + - uses: ./.github/actions/install-nix-action with: - install_url: "https://releases.nixos.org/nix/nix-2.29.1/install" + github_token: ${{ secrets.GITHUB_TOKEN }} + dogfood: true # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: | sandbox = true From 3b3c02160dce1110ed9856aa6234fd37fa5c9347 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 13 Jul 2025 16:05:05 +0300 Subject: [PATCH 08/20] ci: Dogfood nix from master for `vm_tests` and `flake_regressions` This should provide more coverage for the build from master that is being dogfooded. --- .github/workflows/ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2531ee020..da6f35907 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,7 +182,12 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main + - uses: ./.github/actions/install-nix-action + with: + dogfood: true + extra_nix_config: + experimental-features = nix-command flakes + github_token: ${{ secrets.GITHUB_TOKEN }} - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ @@ -208,6 +213,11 @@ jobs: with: repository: NixOS/flake-regressions-data path: flake-regressions/tests - - uses: DeterminateSystems/nix-installer-action@main + - uses: ./.github/actions/install-nix-action + with: + dogfood: true + extra_nix_config: + experimental-features = nix-command flakes + github_token: ${{ secrets.GITHUB_TOKEN }} - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From 6abc29bba526c426bdaf4477d0740d877ac53294 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jul 2025 15:17:33 +0200 Subject: [PATCH 09/20] Move boost version check to libutil --- src/libexpr/meson.build | 1 - src/libutil/meson.build | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 43e4b9c98..f5adafae0 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -41,7 +41,6 @@ boost = dependency( 'boost', modules : ['container', 'context'], include_type: 'system', - version: '>=1.82.0' ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. diff --git a/src/libutil/meson.build b/src/libutil/meson.build index f5ad2b1f6..f48c8f3d7 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -59,6 +59,7 @@ boost = dependency( 'boost', modules : ['context', 'coroutine', 'iostreams'], include_type: 'system', + version: '>=1.82.0' ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. From fde606887430344dd91dbbf907b8cc3bad1f2125 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jul 2025 18:09:06 +0200 Subject: [PATCH 10/20] Improve rendering of ignored exceptions Instead of error (ignored): error: SQLite database '...' is busy we now get error (ignored): SQLite database '...' is busy --- src/libutil/util.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c9cc80fef..23dafe8c9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -190,8 +190,10 @@ void ignoreExceptionInDestructor(Verbosity lvl) try { try { throw; + } catch (Error & e) { + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what()); } } catch (...) { } } @@ -202,8 +204,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl) throw; } catch (const Interrupted & e) { throw; + } catch (Error & e) { + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what()); } } From 7b2f24d68837b178015020f577a4b3abe2d99b92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jul 2025 18:10:07 +0200 Subject: [PATCH 11/20] Improve handleSQLiteBusy() message Closes https://github.com/NixOS/nix/pull/10319. --- src/libstore/sqlite.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 55b967ed6..c3fb1f413 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -250,7 +250,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) if (now > nextWarning) { nextWarning = now + 10; logWarning({ - .msg = HintFmt(e.what()) + .msg = e.info().msg }); } From 8e8416387c935f2dec5bd24ceac3e3553cae0d59 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Fri, 11 Jul 2025 09:34:06 -0700 Subject: [PATCH 12/20] Add error message when git returns non-0 for fetch Users have complained that fetchGit is flaky however the culprit is likely that `git fetch` was unable itself to download the repository for whatever reason (i.e. poor network etc..) Nothing was checking the status of `git fetch` and the error message that would eventually surface to the users were that the commit was not found. Add explicit error checking for status code from `git fetch` and return a message earlier on to indicate that the failure was from that point. fixes #10431 --- src/libfetchers/git-utils.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 9fe271fe8..563c2180d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -545,7 +545,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this append(gitArgs, {"--depth", "1"}); append(gitArgs, {std::string("--"), url, refspec}); - runProgram(RunOptions { + auto [status, output] = runProgram(RunOptions { .program = "git", .lookupPath = true, // FIXME: git stderr messes up our progress indicator, so @@ -554,6 +554,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this .input = {}, .isInteractive = true }); + + if (status > 0) { + throw Error("Failed to fetch git repository %s: %s", url, output); + } } void verifyCommit( From fb6f494d35656556465f05db9f4ee8dbdb1d3bf5 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Fri, 11 Jul 2025 09:39:34 -0700 Subject: [PATCH 13/20] merge stderr to stdout so we can emit it --- src/libfetchers/git-utils.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 563c2180d..d76f6879d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -552,11 +552,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // we're using --quiet for now. Should process its stderr. .args = gitArgs, .input = {}, + .mergeStderrToStdout = true, .isInteractive = true }); - + if (status > 0) { - throw Error("Failed to fetch git repository %s: %s", url, output); + throw Error("Failed to fetch git repository %s : %s", url, output); } } From a4f548fed1b6fe7c7f1882c3214175d4147ddfe4 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Fri, 11 Jul 2025 13:28:04 -0700 Subject: [PATCH 14/20] Fix FetchGit test --- src/libfetchers/git.cc | 14 +++++++++++-- tests/functional/fetchGit.sh | 39 +++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index cf255c001..88fe2e83d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -444,7 +444,11 @@ struct GitInputScheme : InputScheme // repo, treat as a remote URI to force a clone. static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); - bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); + + // Why are we checking for bare repository? + // well if it's a bare repository we want to force a git fetch rather than copying the folder + bool isBareRepository = url.scheme == "file" && pathExists(url.path) && + !pathExists(url.path + "/.git"); // // FIXME: here we turn a possibly relative path into an absolute path. // This allows relative git flake inputs to be resolved against the @@ -462,6 +466,12 @@ struct GitInputScheme : InputScheme "See https://github.com/NixOS/nix/issues/12281 for details.", url); } + + // If we don't check here for the path existence, then we can give libgit2 any directory + // and it will initialize them as git directories. + if (!pathExists(url.path)) { + throw Error("The path '%s' does not exist.", url.path); + } repoInfo.location = std::filesystem::absolute(url.path); } else { if (url.scheme == "file") @@ -599,7 +609,7 @@ struct GitInputScheme : InputScheme ? cacheDir / ref : cacheDir / "refs/heads" / ref; - bool doFetch; + bool doFetch = false; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index a41aa35c0..dc5d8f818 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -53,6 +53,27 @@ rm -rf $TEST_HOME/.cache/nix path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] +# Fetch again. This should be cached. +# NOTE: This has to be done before the test case below which tries to pack-refs +# the reason being that the lookup on the cache uses the ref-file `/refs/heads/master` +# which does not exist after packing. +mv $repo ${repo}-tmp +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]] + +# Fetching with a explicit hash should succeed. +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") +[[ $(cat $path2/hello) = utrecht ]] + +mv ${repo}-tmp $repo + # Fetch when the cache has packed-refs # Regression test of #8822 git -C $TEST_HOME/.cache/nix/gitv3/*/ pack-refs --all @@ -83,24 +104,6 @@ path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \" # But without a hash, it fails. expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'fetchGit' doesn't fetch unlocked input" -# Fetch again. This should be cached. -mv $repo ${repo}-tmp -path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") -[[ $path = $path2 ]] - -[[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]] - -# Fetching with a explicit hash should succeed. -path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path2 ]] - -path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") -[[ $(cat $path2/hello) = utrecht ]] - -mv ${repo}-tmp $repo - # Using a clean working tree should produce the same result. path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $path = $path2 ]] From 196c21c5a0e2f9b3149689ea36789cf0478d893a Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Wed, 16 Jul 2025 21:09:59 -0700 Subject: [PATCH 15/20] Add helpful messages when file:// used as tarball When `file://` is used accidentally in a flake as the source it is expected to be a tarball by default. Add some friendlier error messages to either inform the user this is not in fact a tarball or if it's a git directory, let them know they can use `git+file`. fixes #12935 --- src/libfetchers/tarball.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b0822cc33..59316eabd 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -111,6 +111,25 @@ static DownloadTarballResult downloadTarball_( const Headers & headers, const std::string & displayPrefix) { + + // Some friendly error messages for common mistakes. + // Namely lets catch when the url is a local file path, but + // it is not in fact a tarball. + if (url.rfind("file://", 0) == 0) { + // Remove "file://" prefix to get the local file path + std::string localPath = url.substr(7); + if (!std::filesystem::exists(localPath)) { + throw Error("tarball '%s' does not exist.", localPath); + } + if (std::filesystem::is_directory(localPath)) { + if (std::filesystem::exists(localPath + "/.git")) { + throw Error( + "tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath); + } + throw Error("tarball '%s' is a directory, not a file.", localPath); + } + } + Cache::Key cacheKey{"tarball", {{"url", url}}}; auto cached = settings.getCache()->lookupExpired(cacheKey); From 6681933643e4e80617d3fdc1cb2ea2358acc9a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 17 Jul 2025 12:26:50 +0200 Subject: [PATCH 16/20] Fix Windows header inclusions for clang-tidy Move windows-error.hh includes inside _WIN32 guards to prevent clang-tidy errors when analyzing these files on non-Windows platforms. --- src/libutil/windows/windows-async-pipe.cc | 5 +++-- src/libutil/windows/windows-error.cc | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index d47930a1b..da47c37a8 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,8 @@ -#include "nix/util/windows-async-pipe.hh" -#include "nix/util/windows-error.hh" + #ifdef _WIN32 +# include "nix/util/windows-async-pipe.hh" +# include "nix/util/windows-error.hh" namespace nix::windows { diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 1e7aff830..0761bdfd5 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,6 +1,5 @@ -#include "nix/util/windows-error.hh" - #ifdef _WIN32 +#include "nix/util/windows-error.hh" #include #define WIN32_LEAN_AND_MEAN #include From f12f96bcbbd5edba64c71e1ea8e45a388490d80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 17 Jul 2025 11:17:59 +0200 Subject: [PATCH 17/20] Fix virtual method calls during construction in S3BinaryCacheStoreImpl Move init() call from constructor to openStore() method to avoid calling virtual methods during object construction. This prevents undefined behavior when virtual methods are called before the object is fully constructed. --- src/libstore/s3-binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index cbb47c063..9bb47a010 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -289,8 +289,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); - - init(); } std::string getUri() override @@ -597,10 +595,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore ref S3BinaryCacheStoreImpl::Config::openStore() const { - return make_ref(ref{ + auto store = make_ref(ref{ // FIXME we shouldn't actually need a mutable config std::const_pointer_cast(shared_from_this()) }); + store->init(); + return store; } static RegisterStoreImplementation regS3BinaryCacheStore; From 44963da7872821983a1b28e79a31c9ea305d0830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 17 Jul 2025 11:15:51 +0200 Subject: [PATCH 18/20] Fix virtual method calls during construction in LocalBinaryCacheStore Move init() call from constructor to openStore() method to avoid calling virtual methods during object construction. This prevents undefined behavior when virtual methods are called before the object is fully constructed. --- src/libstore/local-binary-cache-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 2f23135fa..03a9bd055 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -39,7 +39,6 @@ struct LocalBinaryCacheStore : , BinaryCacheStore{*config} , config{config} { - init(); } void init() override; @@ -126,10 +125,12 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() } ref LocalBinaryCacheStoreConfig::openStore() const { - return make_ref(ref{ + auto store = make_ref(ref{ // FIXME we shouldn't actually need a mutable config std::const_pointer_cast(shared_from_this()) }); + store->init(); + return store; } static RegisterStoreImplementation regLocalBinaryCacheStore; From d678b071d69569786db4a4cc8110ee0cd4496e2f Mon Sep 17 00:00:00 2001 From: Oleksandr Knyshuk Date: Thu, 17 Jul 2025 17:26:56 +0200 Subject: [PATCH 19/20] Make nix help shell work by handling aliases properly Previously, `nix help shell` failed with "Nix has no subcommand 'shell'" despite `nix shell --help` working correctly. This happened because the `shell` command is actually an alias for `env shell`, and the help system wasn't resolving aliases when looking up documentation. This patch modifies the `showHelp` function to check for and resolve aliases before generating the manpage name, ensuring that shorthand commands like `shell` get proper help documentation. Closes: #13431 --- src/nix/main.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 6144f746f..502e04e60 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -212,6 +212,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs lowdown. */ static void showHelp(std::vector subcommand, NixArgs & toplevel) { + // Check for aliases if subcommand has exactly one element + if (subcommand.size() == 1) { + auto alias = toplevel.aliases.find(subcommand[0]); + if (alias != toplevel.aliases.end()) { + subcommand = alias->second.replacement; + } + } + auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); evalSettings.restrictEval = false; From cfb8a318853d95d5a5346c1ad5a975c0c07fed26 Mon Sep 17 00:00:00 2001 From: Oleksandr Knyshuk Date: Thu, 17 Jul 2025 21:58:15 +0200 Subject: [PATCH 20/20] Require rsync in nix-manual meson.build Closes: #13313 --- doc/manual/meson.build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fe2374a7..0779cd267 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -8,6 +8,7 @@ nix = find_program('nix', native : true) mdbook = find_program('mdbook', native : true) bash = find_program('bash', native : true) +rsync = find_program('rsync', required: true, native: true) pymod = import('python') python = pymod.find_installation('python3') @@ -84,7 +85,7 @@ manual = custom_target( @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml - rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + @4@ -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 rm -rf @2@/manual mv @2@/html @2@/manual @@ -94,6 +95,7 @@ manual = custom_target( mdbook.full_path(), meson.current_build_dir(), meson.project_version(), + rsync.full_path(), ), ], input : [