diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml new file mode 100644 index 000000000..b97076bd7 --- /dev/null +++ b/.github/workflows/hydra_status.yml @@ -0,0 +1,16 @@ +name: Hydra status +on: + schedule: + - cron: "12,42 * * * *" + workflow_dispatch: +jobs: + check_hydra_status: + name: Check Hydra status + if: github.repository_owner == 'NixOS' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - run: bash scripts/check-hydra-status.sh + diff --git a/.version b/.version index 68151b2e1..fad066f80 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.5 \ No newline at end of file +2.5.0 \ No newline at end of file diff --git a/configure.ac b/configure.ac index 65478ecc5..c35065704 100644 --- a/configure.ac +++ b/configure.ac @@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) ]) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) # Look for libcpuid. +have_libcpuid= if test "$machine_name" = "x86_64"; then - PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"]) - have_libcpuid=1 - AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid]) + AC_ARG_ENABLE([cpuid], + AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)])) + if test "x$enable_cpuid" != "xno"; then + PKG_CHECK_MODULES([LIBCPUID], [libcpuid], + [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS" + have_libcpuid=1 + AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])] + ) + fi fi AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index e43d9f2fb..6b232a736 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -12,11 +12,13 @@ man-pages := $(foreach n, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. +# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ - NIX_STATE_DIR=/dummy + NIX_STATE_DIR=/dummy \ + NIX_CONFIG='cores = 0' nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2a1e55e5b..90a8f1f94 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found: $ nix-shell ``` +or if you have a flake-enabled nix: + +```console +$ nix develop +``` + +To get a shell with a different compilation environment (e.g. stdenv, +gccStdenv, clangStdenv, clang11Stdenv): + +```console +$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +``` + +or if you have a flake-enabled nix: + +```console +$ nix develop .#clang11StdenvPackages +``` + To build Nix itself in this shell: ```console diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index bb350d9de..71ff13275 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -47,7 +47,7 @@ the store object at `P` contains the path `Q` somewhere. The *references* of a store path are the set of store paths to which it has a reference. - + A derivation can reference other derivations and sources (but not output paths), whereas an output path only references other output paths. @@ -66,7 +66,7 @@ is necessary to deploy whole closures, since otherwise at runtime files could be missing. The command `nix-store -qR` prints out closures of store paths. - + As an example, if the store object at path `P` contains a reference to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. @@ -98,3 +98,7 @@ store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + - `∅` \ + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + - `ε` \ + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 96fa34635..4367654a2 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble: - update `/etc/synthetic.conf` to direct macOS to create a "synthetic" empty root directory to mount your volume - specify mount options for the volume in `/etc/fstab` + - `rw`: read-write + - `noauto`: prevent the system from auto-mounting the volume (so the + LaunchDaemon mentioned below can control mounting it, and to avoid + masking problems with that mounting service). + - `nobrowse`: prevent the Nix Store volume from showing up on your + desktop; also keeps Spotlight from spending resources to index + this volume + - if you have FileVault enabled - generate an encryption password - put it in your system Keychain diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 3d2255b7a..ee7614bdb 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,7 +3,7 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker -ti run nixos/nix +$ docker run -ti nixos/nix Unable to find image 'nixos/nix:latest' locally latest: Pulling from nixos/nix 5843afab3874: Pull complete diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 0323a4f55..6f4eb3008 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -44,6 +44,11 @@ obtained from the its repository . + - The `libsodium` library for verifying cryptographic signatures + of contents fetched from binary caches. + It can be obtained from the official web site + . + - Recent versions of Bison and Flex to build the parser. (This is because Nix needs GLR support in Bison and reentrancy support in Flex.) For Bison, you need version 2.6, which can be obtained from @@ -58,3 +63,11 @@ `--disable-seccomp-sandboxing` option to the `configure` script (Not recommended unless your system doesn't support `libseccomp`). To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library + is used to determine which microarchitecture levels are supported + (e.g., as whether to have `x86_64-v2-linux` among additional system types). + The library is available from its homepage + . + This is an optional dependency and can be disabled + by providing a `--disable-cpuid` to the `configure` script. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 26c7d2cce..cf2da9aa8 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -5,3 +5,8 @@ * `nix develop` now has a flag `--unpack` to run `unpackPhase`. * Lists can now be compared lexicographically using the `<` operator. + +* New built-in function: `builtins.groupBy`, with the same functionality as + Nixpkgs' `lib.groupBy`, but faster. + +* `nix repl` now has a `:log` command. diff --git a/flake.nix b/flake.nix index ff152ebd6..9a5d188ae 100644 --- a/flake.nix +++ b/flake.nix @@ -22,15 +22,36 @@ crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + forAllSystemsAndStdenvs = f: forAllSystems (system: + nixpkgs.lib.listToAttrs + (map + (n: + nixpkgs.lib.nameValuePair "${n}Packages" ( + f system n + )) stdenvs + ) + ); + + forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = forAllSystems (system: - import nixpkgs { - inherit system; - overlays = [ self.overlay ]; - } - ); + nixpkgsFor = + let stdenvsPackages = forAllSystemsAndStdenvs + (system: stdenv: + import nixpkgs { + inherit system; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + } + ); + in + # Add the `stdenvPackages` at toplevel, both because these are the ones + # we want most of the time and for backwards compatibility + forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); commonDeps = pkgs: with pkgs; rec { # Use "busybox-sandbox-shell" if present, @@ -255,18 +276,15 @@ $(cat ${installerClosureInfo}/store-paths) ''; - in { - - # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. - overlay = final: prev: { - + overlayFor = getStdenv: final: prev: + let currentStdenv = getStdenv final; in + { nixStable = prev.nix; # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - nix = with final; with commonDeps pkgs; stdenv.mkDerivation { + nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -288,9 +306,9 @@ mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' + ${lib.optionalString currentStdenv.isLinux '' chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} ''; @@ -317,7 +335,7 @@ strictDeps = true; - passthru.perl-bindings = with final; stdenv.mkDerivation { + passthru.perl-bindings = with final; currentStdenv.mkDerivation { name = "nix-perl-${version}"; src = self; @@ -336,8 +354,8 @@ pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium + ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} @@ -351,7 +369,7 @@ }; - lowdown-nix = with final; stdenv.mkDerivation rec { + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; src = lowdown-src; @@ -361,15 +379,20 @@ nativeBuildInputs = [ buildPackages.which ]; configurePhase = '' - ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} + ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin - ''; + ''; }; - }; + in { + + # A Nixpkgs overlay that overrides the 'nix' and + # 'nix.perl-bindings' packages. + overlay = overlayFor (p: p.stdenv); + hydraJobs = { # Binary package for various platforms. @@ -610,15 +633,22 @@ doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; }; - }) crossSystems))); + }) crossSystems)) // (builtins.listToAttrs (map (stdenvName: + nixpkgsFor.${system}.lib.nameValuePair + "nix-${stdenvName}" + nixpkgsFor.${system}."${stdenvName}Packages".nix + ) stdenvs)) + ); defaultPackage = forAllSystems (system: self.packages.${system}.nix); - devShell = forAllSystems (system: + devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages); + + devShells = forAllSystemsAndStdenvs (system: stdenv: with nixpkgsFor.${system}; with commonDeps pkgs; - stdenv.mkDerivation { + nixpkgsFor.${system}.${stdenv}.mkDerivation { name = "nix"; outputs = [ "out" "dev" "doc" ]; diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index f1b439840..da1970f69 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -25,5 +25,10 @@ /var/log/nix-daemon.log StandardOutPath /dev/null + SoftResourceLimits + + NumberOfFiles + 4096 + diff --git a/perl/configure.ac b/perl/configure.ac index eb65ac17b..a02cb06c9 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};') AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh new file mode 100644 index 000000000..c1d2d7c40 --- /dev/null +++ b/scripts/check-hydra-status.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail +# set -x + + +# mapfile BUILDS_FOR_LATEST_EVAL < <( +# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ +# jq -r '.evals[0].builds[] | @sh') +BUILDS_FOR_LATEST_EVAL=$( +curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ + jq -r '.evals[0].builds[]') + +someBuildFailed=0 + +for buildId in $BUILDS_FOR_LATEST_EVAL; do + buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") + + buildStatus=$(echo "$buildInfo" | \ + jq -r '.buildstatus') + + if [[ "$buildStatus" -ne 0 ]]; then + someBuildFailed=1 + echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra" + fi +done + +exit "$someBuildFailed" diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 334b75045..bd8a7ee3a 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -440,7 +440,22 @@ add_nix_vol_fstab_line() { # shellcheck disable=SC1003,SC2026 local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" < "$SCRATCH/ex_cleanroom_wrapper" <&2 - _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" < /dev/null >&2; then warning <&2 + echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2 fi case "$(uname -s)" in @@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do echo " providing multi-user support and better isolation for local builds." echo " Both for security and reproducibility, this method is recommended if" echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo " See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" echo "" echo " --no-daemon: Simple, single-user installation that does not require root and is" echo " trivial to uninstall." @@ -144,7 +144,7 @@ if ! [ -e "$dest" ]; then fi if ! [ -w "$dest" ]; then - echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b1bdae39e..f5ac09261 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -291,6 +291,9 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref store, std::string_view prefix) { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + return; + if (prefix == "") completions->add("."); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index aa496abb1..73f5242fc 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -482,11 +482,16 @@ LockedFlake lockFlake( } } + LockParent newParent { + .path = inputPath, + .absolute = false + }; + computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs : fakeInputs, - childNode, inputPath, oldLock, parent, parentPath); + childNode, inputPath, oldLock, newParent, parentPath); } else { /* We need to create a new lock file entry. So fetch diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c0d59da8c..66af373d7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2928,6 +2928,56 @@ static RegisterPrimOp primop_partition({ .fun = prim_partition, }); +static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + ValueVectorMap attrs; + + for (auto vElem : args[1]->listItems()) { + Value res; + state.callFunction(*args[0], *vElem, res, pos); + string name = state.forceStringNoCtx(res, pos); + Symbol sym = state.symbols.create(name); + auto vector = attrs.try_emplace(sym, ValueVector()).first; + vector->second.push_back(vElem); + } + + state.mkAttrs(v, attrs.size()); + + for (auto & i : attrs) { + Value * list = state.allocAttr(v, i.first); + auto size = i.second.size(); + state.mkList(*list, size); + memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size); + } +} + +static RegisterPrimOp primop_groupBy({ + .name = "__groupBy", + .args = {"f", "list"}, + .doc = R"( + Groups elements of *list* together by the string returned from the + function *f* called on each element. It returns an attribute set + where each attribute value contains the elements of *list* that are + mapped to the same corresponding attribute name returned by *f*. + + For example, + + ```nix + builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"] + ``` + + evaluates to + + ```nix + { b = [ "bar" "baz" ]; f = [ "foo" ]; } + ``` + )", + .fun = prim_groupBy, +}); + static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -3732,7 +3782,7 @@ void EvalState::createBaseEnv() .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), .name = symbols.create(primOp.name), - .args = std::move(primOp.args), + .args = primOp.args, .doc = primOp.doc, }); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 4b43e47ae..6b4f3c0ae 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -425,9 +425,11 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC typedef std::vector > ValueVector; typedef std::map, traceable_allocator > > ValueMap; +typedef std::map, traceable_allocator > > ValueVectorMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; +typedef std::map ValueVectorMap; #endif diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fb5702c4c..07e543c53 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme // for security, ensure that if the parent is a store path, it's inside it if (store->isInStore(parent)) { auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isInDir(absPath, storePath)) + if (!isDirOrInDir(absPath, storePath)) throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); } } else diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 7cf88ec80..46390406f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -20,7 +20,7 @@ ProgressBarSettings progressBarSettings; static GlobalConfig::Register rProgressBarSettings(&progressBarSettings); -static std::string getS(const std::vector & fields, size_t n) +static std::string_view getS(const std::vector & fields, size_t n) { assert(n < fields.size()); assert(fields[n].type == Logger::Field::tString); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index b6bfea8cb..4404e0195 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -427,7 +427,7 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - + stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -438,7 +438,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - close(STDOUT_FILENO); + dup2(stdout, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 05277d90a..ed012959b 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -88,6 +88,7 @@ public: private: Pid pid; + int stdout; }; extern volatile ::sig_atomic_t blockInt; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 694b1f5ea..8ecff4775 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -656,7 +656,7 @@ void DerivationGoal::tryLocalBuild() { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 8c10dbbef..8d51ed023 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -140,6 +140,8 @@ void PathSubstitutionGoal::tryNext() { warn("substituter '%s' does not have a valid signature for path '%s'", sub->getUri(), worker.store.printStorePath(storePath)); + warn("verify that your nix.conf contains a correct signature in 'trusted-public-keys' for %s", + sub->getUri()); tryNext(); return; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 92f7e0f0d..962beea55 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -291,11 +291,11 @@ void Worker::run(const Goals & _topGoals) if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); else throw Error("unable to start any build; remote machines may not have " "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } assert(!awake.empty()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2eb566080..dc4889dfd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -431,25 +431,30 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgo = parseHashType(hashAlgoRaw); } - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - + auto dumpSource = sinkToSource([&](Sink & saved) { + if (method == FileIngestionMethod::Recursive) { + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else { + /* Incrementally parse the NAR file, stripping the + metadata, and streaming the sole file we expect into + `saved`. */ + RetrieveRegularNARSink savedRegular { saved }; + parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); + } + }); logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 64019314f..3a1688272 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -8,6 +8,7 @@ #include "references.hh" #include "callback.hh" #include "topo-sort.hh" +#include "finally.hh" #include #include @@ -1333,13 +1334,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; + Finally cleanup([&]() { + dump.resize(oldSize + got); + }); try { got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; } - dump.resize(oldSize + got); } std::unique_ptr delTempDir; diff --git a/src/libstore/names.cc b/src/libstore/names.cc index ce808accc..54c95055d 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -42,7 +42,7 @@ DrvName::~DrvName() { } -bool DrvName::matches(DrvName & n) +bool DrvName::matches(const DrvName & n) { if (name != "*") { if (!regex) { diff --git a/src/libstore/names.hh b/src/libstore/names.hh index bc62aac93..3f861bc44 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -19,7 +19,7 @@ struct DrvName DrvName(std::string_view s); ~DrvName(); - bool matches(DrvName & n); + bool matches(const DrvName & n); private: std::unique_ptr regex; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a627e9cf1..7f7e973e9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -684,6 +684,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, Callback> callback) noexcept { auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + try { + callback(nullptr); + } catch (...) { return callback.rethrow(); } + } + conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index defb77a10..1b6467eb2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -512,6 +512,7 @@ std::pair createTempFile(const Path & prefix) AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); return {std::move(fd), tmpl}; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0bdb37a79..bc96bfed1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 75576ef8a..e2325c91f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -359,6 +359,7 @@ static void main_nix_build(int argc, char * * argv) is not set, then build bashInteractive from . */ auto shell = getEnv("NIX_BUILD_SHELL"); + std::optional shellDrv; if (!shell) { @@ -375,8 +376,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = store->parseStorePath(drv->queryDrvPath()); pathsToBuild.push_back({bashDrv}); pathsToCopy.insert(bashDrv); - - shell = drv->queryOutPath() + "/bin/bash"; + shellDrv = bashDrv; } catch (Error & e) { logError(e.info()); @@ -402,6 +402,11 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (shellDrv) { + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; + } + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0dd0e34a0..b9e7be1c6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors) } +namespace { + +std::set searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { + constexpr std::size_t maxResults = 3; + std::set result; + for (const auto & drvInfo : allElems) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (hasPrefix(drvName.name, prefix)) { + result.emplace(drvName.name); + + if (result.size() >= maxResults) { + break; + } + } + } + return result; +} + +struct Match +{ + DrvInfo drvInfo; + std::size_t index; + + Match(DrvInfo drvInfo_, std::size_t index_) + : drvInfo{std::move(drvInfo_)} + , index{index_} + {} +}; + +/* If a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ +std::vector pickNewestOnly(EvalState & state, std::vector matches) { + /* Map from package names to derivations. */ + std::map newest; + StringSet multiple; + + for (auto & match : matches) { + auto & oneDrv = match.drvInfo; + + const auto drvName = DrvName { oneDrv.queryName() }; + long comparison = 1; + + const auto itOther = newest.find(drvName.name); + + if (itOther != newest.end()) { + auto & newestDrv = itOther->second.drvInfo; + + comparison = + oneDrv.querySystem() == newestDrv.querySystem() ? 0 : + oneDrv.querySystem() == settings.thisSystem ? 1 : + newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + if (comparison == 0) + comparison = comparePriorities(state, oneDrv, newestDrv); + if (comparison == 0) + comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version); + } + + if (comparison > 0) { + newest.erase(drvName.name); + newest.emplace(drvName.name, match); + multiple.erase(drvName.fullName); + } else if (comparison == 0) { + multiple.insert(drvName.fullName); + } + } + + matches.clear(); + for (auto & [name, match] : newest) { + if (multiple.find(name) != multiple.end()) + warn( + "there are multiple derivations named '%1%'; using the first one", + name); + matches.push_back(match); + } + + return matches; +} + +} // end namespace + static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, const Strings & args, bool newestOnly) { @@ -232,79 +317,42 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, selectors.emplace_back("*"); DrvInfos elems; - set done; + std::set done; - for (auto & i : selectors) { - typedef list > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->queryName()); - if (i.matches(drvName)) { - i.hits++; - matches.push_back(std::pair(*j, n)); + for (auto & selector : selectors) { + std::vector matches; + for (const auto & [index, drvInfo] : enumerate(allElems)) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (selector.matches(drvName)) { + ++selector.hits; + matches.emplace_back(drvInfo, index); } } - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map > Newest; - Newest newest; - StringSet multiple; - - for (auto & j : matches) { - DrvName drvName(j.first.queryName()); - long d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j.first.querySystem() == k->second.first.querySystem() ? 0 : - j.first.querySystem() == settings.thisSystem ? 1 : - k->second.first.querySystem() == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j.first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.queryName()); - } else if (d == 0) { - multiple.insert(j.first.queryName()); - } - } - - matches.clear(); - for (auto & j : newest) { - if (multiple.find(j.second.first.queryName()) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", - j.second.first.queryName()); - matches.push_back(j.second); - } + matches = pickNewestOnly(state, std::move(matches)); } /* Insert only those elements in the final list that we haven't inserted before. */ - for (auto & j : matches) - if (done.insert(j.second).second) - elems.push_back(j.first); - } + for (auto & match : matches) + if (done.insert(match.index).second) + elems.push_back(match.drvInfo); - checkSelectorUse(selectors); + if (selector.hits == 0 && selector.fullName != "*") { + const auto prefixHits = searchByPrefix(allElems, selector.name); + + if (prefixHits.empty()) { + throw Error("selector '%1%' matches no derivations", selector.fullName); + } else { + std::string suggestionMessage = ", maybe you meant:"; + for (const auto & drvName : prefixHits) { + suggestionMessage += fmt("\n%s", drvName); + } + throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName); + } + } + } return elems; } diff --git a/src/nix/registry.md b/src/nix/registry.md index a1674bd2e..d5c9ef442 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -2,7 +2,7 @@ R""( # Description -`nix flake` provides subcommands for managing *flake +`nix registry` provides subcommands for managing *flake registries*. Flake registries are a convenience feature that allows you to refer to flakes using symbolic identifiers such as `nixpkgs`, rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 41283c5f2..42143871f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -279,6 +279,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) }; setupSignals(); + Finally resetTerminal([&]() { rl_deprep_terminal(); }); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); restoreSignals(); @@ -356,6 +357,8 @@ StringSet NixRepl::completePrefix(string prefix) // Quietly ignore evaluation errors. } catch (UndefinedVarError & e) { // Quietly ignore undefined variable errors. + } catch (BadURL & e) { + // Quietly ignore BadURL flake-related errors. } } @@ -427,7 +430,8 @@ bool NixRepl::processLine(string line) << " :s Build dependencies of derivation, then start nix-shell\n" << " :t Describe result of evaluation\n" << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n"; + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n"; } else if (command == ":a" || command == ":add") { @@ -497,7 +501,7 @@ bool NixRepl::processLine(string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":i" || command == ":s") { + else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); @@ -511,6 +515,27 @@ bool NixRepl::processLine(string line) logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); + } else if (command == ":log") { + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto log = sub->getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); } else { runNix("nix-shell", {drvPathRaw}); } diff --git a/src/nix/repl.md b/src/nix/repl.md index bba60f871..9b6f2bee3 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -35,14 +35,17 @@ R""( nix-repl> emacs.drvPath "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" - nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out" - nix-repl> :b x + nix-repl> :b drv this derivation produced the following outputs: out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello nix-repl> builtins.readFile drv "Hello, world!\n" + + nix-repl> :log drv + Hello, world! ``` # Description diff --git a/tests/ca-shell.nix b/tests/ca-shell.nix new file mode 100644 index 000000000..ad2ab6aff --- /dev/null +++ b/tests/ca-shell.nix @@ -0,0 +1 @@ +{ ... }@args: import ./shell.nix (args // { contentAddressed = true; }) diff --git a/tests/flakes.sh b/tests/flakes.sh index 9e10322b9..20966ab2a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -722,6 +722,7 @@ cat > $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix <