Merge pull request #5650 from obsidiansystems/2.3-alt-daemon-test-suite

[backport 2.3] Allow testing with different daemons

mk/run_test.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-set -u
+set -eu -o pipefail
 
 red=""
 green=""
@@ -15,8 +15,9 @@ if [ -t 1 ]; then
     normal=""
 fi
 (cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
-log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+
-status=$?
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" && status=0 || status=$?
+
 if [ $status -eq 0 ]; then
   echo "$post_run_msg [${green}PASS$normal]"
 elif [ $status -eq 99 ]; then

mk/tests.mk

@@ -13,3 +13,6 @@ define run-install-test
 endef
 
 .PHONY: check installcheck
+
+print-top-help += \
+  echo "  installcheck: Run functional tests";

release.nix

@@ -223,8 +223,42 @@ let
         FONTCONFIG_FILE = texFunctions.fontsConf;
       };
 
+    testNixVersions = client: daemon: pkgs:
+
+      with import ./release-common.nix { inherit pkgs; };
+
+      pkgs.releaseTools.nixBuild {
+        NIX_DAEMON_PACKAGE = daemon;
+        NIX_CLIENT_PACKAGE = client;
+        HAVE_LOCAL_NIX_BUILD = false;
+        name = "nix-tests-${client.src.version}-against-${daemon.src.version}";
+
+        src = tarball;
+
+        VERSION_SUFFIX = tarball.versionSuffix;
+
+        buildInputs = buildDeps;
+        propagatedBuildInputs = propagatedDeps;
+
+        enableParallelBuilding = true;
+
+        dontBuild = true;
+        doInstallCheck = true;
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+        installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORE";
+      };
+
+    testAgainstSelf = pkgs.lib.genAttrs systems (system:
+
+      let pkgs = import nixpkgs { inherit system; }; in
+
+      testNixVersions build.${system} build.${system} pkgs);
 
     # System tests.
+
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
       inherit nixpkgs;
       nix = build.x86_64-linux; system = "x86_64-linux";

tests/binary-cache.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“--no-require-sigs” can’t be used with the daemon"
+
 clearStore
 clearCache
 
@@ -60,8 +62,8 @@ mv $nar $nar.good
 mkdir -p $TEST_ROOT/empty
 nix-store --dump $TEST_ROOT/empty | xz > $nar
 
-nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
-grep -q "hash mismatch" $TEST_ROOT/log
+grepQuiet "hash mismatch" $TEST_ROOT/log
 
 mv $nar.good $nar
 
@@ -108,7 +110,7 @@ clearStore
 rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
-grep -q "copying path" $TEST_ROOT/log
+grepQuiet "copying path" $TEST_ROOT/log
 
 
 if [ -n "$HAVE_SODIUM" ]; then

tests/build-dry.sh

@@ -20,7 +20,7 @@ nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
 # TODO: XXX: FIXME: #1793
 # Disable this part of the test until the problem is resolved:
-if [ -n "$ISSUE_1795_IS_FIXED" ]; then
+if [ -n "${ISSUE_1795_IS_FIXED-}" ]; then
 clearStore
 clearCache
 

tests/check-refs.sh

@@ -8,14 +8,14 @@ dep=$(nix-build -o $RESULT check-refs.nix -A dep)
 
 # test1 references dep, not itself.
 test1=$(nix-build -o $RESULT check-refs.nix -A test1)
-(! nix-store -q --references $test1 | grep -q $test1)
+nix-store -q --references $test1 | grepQuietInverse $test1
-nix-store -q --references $test1 | grep -q $dep
+nix-store -q --references $test1 | grepQuiet $dep
 
 # test2 references src, not itself nor dep.
 test2=$(nix-build -o $RESULT check-refs.nix -A test2)
-(! nix-store -q --references $test2 | grep -q $test2)
+nix-store -q --references $test2 | grepQuietInverse $test2
-(! nix-store -q --references $test2 | grep -q $dep)
+nix-store -q --references $test2 | grepQuietInverse $dep
-nix-store -q --references $test2 | grep -q aux-ref
+nix-store -q --references $test2 | grepQuiet aux-ref
 
 # test3 should fail (unallowed ref).
 (! nix-build -o $RESULT check-refs.nix -A test3)

tests/check-reqs.sh

@@ -8,8 +8,8 @@ nix-build -o $RESULT check-reqs.nix -A test1
 
 (! nix-build -o $RESULT check-reqs.nix -A test2)
 (! nix-build -o $RESULT check-reqs.nix -A test3)
-(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1'
+(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1'
-(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2'
+(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2'
 (! nix-build -o $RESULT check-reqs.nix -A test5)
 (! nix-build -o $RESULT check-reqs.nix -A test6)
 

tests/check.sh

@@ -1,5 +1,8 @@
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+buggyNeedLocalStore "see #4813"
+
 clearStore
 
 nix-build dependencies.nix --no-out-link

tests/common.sh.in

@@ -1,4 +1,10 @@
-set -e
+set -eu -o pipefail
+
+if [[ -z "${COMMON_SH_SOURCED-}" ]]; then
+
+COMMON_SH_SOURCED=1
+
+export PS4='+(${BASH_SOURCE[0]}:$LINENO) '
 
 export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
 export NIX_STORE_DIR
@@ -18,7 +24,7 @@ if [[ -n $NIX_STORE ]]; then
 fi
 export _NIX_IN_TEST=$TEST_ROOT/shared
 export _NIX_TEST_NO_LSOF=1
-export NIX_REMOTE=$NIX_REMOTE_
+export NIX_REMOTE=${NIX_REMOTE_-}
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
@@ -26,6 +32,13 @@ unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 
 export PATH=@bindir@:$PATH
+if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then
+  export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH
+fi
+DAEMON_PATH="$PATH"
+if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+  DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH"
+fi
 coreutils=@coreutils@
 
 export dot=@dot@
@@ -37,6 +50,9 @@ export HAVE_SODIUM="@HAVE_SODIUM@"
 export version=@PACKAGE_VERSION@
 export system=@system@
 
+export IMPURE_VAR1=foo
+export IMPURE_VAR2=bar
+
 cacheDir=$TEST_ROOT/binary-cache
 
 readLink() {
@@ -55,7 +71,6 @@ clearStore() {
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
     mkdir "$NIX_STATE_DIR"
-    nix-store --init
     clearProfiles
 }
 
@@ -68,31 +83,73 @@ clearCacheCache() {
 }
 
 startDaemon() {
+    # Don’t start the daemon twice, as this would just make it loop indefinitely
+    if [[ "$NIX_REMOTE" == daemon ]]; then
+      return
+    fi
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
-    rm -f $NIX_STATE_DIR/daemon-socket/socket
+    rm -f $NIX_DAEMON_SOCKET_PATH
-    nix-daemon &
+    PATH=$DAEMON_PATH nix-daemon &
-    for ((i = 0; i < 30; i++)); do
-        if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
-        sleep 1
-    done
     pidDaemon=$!
-    trap "kill -9 $pidDaemon" EXIT
+    for ((i = 0; i < 300; i++)); do
+        if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
+        sleep 0.1
+    done
+    trap "killDaemon" EXIT
+    # Save for if daemon is killed
+    NIX_REMOTE_OLD=$NIX_REMOTE
     export NIX_REMOTE=daemon
 }
 
 killDaemon() {
-    kill -9 $pidDaemon
+    kill $pidDaemon
+    for i in {0..100}; do
+        kill -0 $pidDaemon || break
+        sleep 0.1
+    done
+    kill -9 $pidDaemon || true
     wait $pidDaemon || true
+    # Restore old nix remote
+    NIX_REMOTE=$NIX_REMOTE_OLD
     trap "" EXIT
 }
 
+restartDaemon() {
+  [[ -z "${pidDaemon:-}" ]] && return 0
+
+  killDaemon
+  startDaemon
+}
+
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi
 
+isDaemonOlder () {
+  [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
+  local requiredVersion="$1"
+  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+  [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -lt 0 ]]
+}
+
+isDaemonNewer () {
+  [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
+  local requiredVersion="$1"
+  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+  [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
+}
+
+requireDaemonOlderThan () {
+  isDaemonOlder "$1" || exit 99
+}
+
+requireDaemonNewerThan () {
+  isDaemonNewer "$1" || exit 99
+}
+
 canUseSandbox() {
-    if [[ ! $_canUseSandbox ]]; then
+    if [[ ! ${_canUseSandbox-} ]]; then
         echo "Sandboxing not supported, skipping this test..."
         return 1
     fi
@@ -105,15 +162,90 @@ fail() {
     exit 1
 }
 
+# Run a command failing if it didn't exit with the expected exit code.
+#
+# Has two advantages over the built-in `!`:
+#
+# 1. `!` conflates all non-0 codes. `expect` allows testing for an exact
+# code.
+#
+# 2. `!` unexpectedly negates `set -e`, and cannot be used on individual
+# pipeline stages with `set -o pipefail`. It only works on the entire
+# pipeline, which is useless if we want, say, `nix ...` invocation to
+# *fail*, but a grep on the error message it outputs to *succeed*.
 expect() {
     local expected res
     expected="$1"
     shift
     set +e
-    "$@"
+    "$@" && res=0 || res="$?"
-    res="$?"
     set -e
-    [[ $res -eq $expected ]]
+    if [[ $res -ne $expected ]]; then
+        echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2
+        return 1
+    fi
+    return 0
+}
+
+# Better than just doing `expect ... >&2` because the "Expected..."
+# message below will *not* be redirected.
+expectStderr() {
+    local expected res
+    expected="$1"
+    shift
+    "$@" 2>&1 && res=0 || res="$?"
+    if [[ $res -ne $expected ]]; then
+        echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2
+        return 1
+    fi
+    return 0
+}
+
+needLocalStore() {
+  if [[ "$NIX_REMOTE" == "daemon" ]]; then
+    echo "Can’t run through the daemon ($1), skipping this test..."
+    return 99
+  fi
+}
+
+# Just to make it easy to find which tests should be fixed
+buggyNeedLocalStore () {
+  needLocalStore "$1"
+}
+
+# `grep -v` doesn't work well for exit codes. We want `!(exist line l. l
+# matches)`. It gives us `exist line l. !(l matches)`.
+#
+# `!` normally doesn't work well with `set -e`, but when we wrap in a
+# function it *does*.
+grepInverse() {
+    ! grep "$@"
+}
+
+# A shorthand, `> /dev/null` is a bit noisy.
+#
+# `grep -q` would seem to do this, no function necessary, but it is a
+# bad fit with pipes and `set -o pipefail`: `-q` will exit after the
+# first match, and then subsequent writes will result in broken pipes.
+#
+# Note that reproducing the above is a bit tricky as it depends on
+# non-deterministic properties such as the timing between the match and
+# the closing of the pipe, the buffering of the pipe, and the speed of
+# the producer into the pipe. But rest assured we've seen it happen in
+# CI reliably.
+grepQuiet() {
+    grep "$@" > /dev/null
+}
+
+# The previous two, combined
+grepQuietInverse() {
+    ! grep "$@" > /dev/null
 }
 
 set -x
+
+if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+    startDaemon
+fi
+
+fi # COMMON_SH_SOURCED

tests/dependencies.sh

@@ -36,10 +36,10 @@ deps=$(nix-store -quR "$drvPath")
 echo "output closure contains $deps"
 
 # The output path should be in the closure.
-echo "$deps" | grep -q "$outPath"
+echo "$deps" | grepQuiet "$outPath"
 
 # Input-1 is not retained.
-if echo "$deps" | grep -q "dependencies-input-1"; then exit 1; fi
+if echo "$deps" | grepQuiet "dependencies-input-1"; then exit 1; fi
 
 # Input-2 is retained.
 input2OutPath=$(echo "$deps" | grep "dependencies-input-2")
@@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath"
 
 # Check that the derivers are set properly.
 test $(nix-store -q --deriver "$outPath") = "$drvPath"
-nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" 
+nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv"

tests/dump-db.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--dump-db requires a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)

tests/export-graph.sh

@@ -4,7 +4,7 @@ clearStore
 clearProfiles
 
 checkRef() {
-    nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1"
+    nix-store -q --references $TEST_ROOT/result | grepQuiet "$1"'$' || fail "missing reference $1"
 }
 
 # Test the export of the runtime dependency graph.

tests/fetchurl.sh

@@ -60,7 +60,7 @@ hash=$(nix-hash --flat --type sha256 $nar)
 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \
           --arg unpack true --argstr name xyzzy --no-out-link)
 
-echo $outPath | grep -q 'xyzzy'
+echo $outPath | grepQuiet 'xyzzy'
 
 test -x $outPath/fetchurl.sh
 test -L $outPath/symlink

tests/fixed.sh

@@ -2,9 +2,6 @@ source common.sh
 
 clearStore
 
-export IMPURE_VAR1=foo
-export IMPURE_VAR2=bar
-
 path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0))
 
 echo 'testing bad...'

tests/function-trace.sh

@@ -10,17 +10,15 @@ expect_trace() {
             --trace-function-calls \
             --expr "$expr" 2>&1 \
             | grep "function-trace" \
-            | sed -e 's/ [0-9]*$//'
+            | sed -e 's/ [0-9]*$//' \
-    );
+            || true
+    )
 
     echo -n "Tracing expression '$expr'"
-    set +e
     msg=$(diff -swB \
                <(echo "$expect") \
                <(echo "$actual")
-    );
+    ) && result=0 || result=$?
-    result=$?
-    set -e
     if [ $result -eq 0 ]; then
         echo " ok."
     else
@@ -81,5 +79,3 @@ function-trace exited undefined position at
 function-trace entered (string):1:1 at
 function-trace exited (string):1:1 at
 "
-
-set -e

tests/gc-auto.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“min-free” and “max-free” are daemon options"
+
 clearStore
 
 garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh)

tests/hash.sh

@@ -2,7 +2,7 @@ source common.sh
 
 try () {
     printf "%s" "$2" > $TEST_ROOT/vector
-    hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector)
+    hash=$(nix hash-file --base16 ${EXTRA-} --type "$1" $TEST_ROOT/vector)
     if test "$hash" != "$3"; then
         echo "hash $1, expected $3, got $hash"
         exit 1

tests/init.sh

@@ -1,8 +1,14 @@
+set -eu -o pipefail
+
 source common.sh
 
 test -n "$TEST_ROOT"
 if test -d "$TEST_ROOT"; then
     chmod -R u+w "$TEST_ROOT"
+    # We would delete any daemon socket, so let's stop the daemon first.
+    if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+        killDaemon
+    fi
     rm -rf "$TEST_ROOT"
 fi
 mkdir "$TEST_ROOT"
@@ -18,6 +24,7 @@ build-users-group =
 keep-derivations = false
 sandbox = false
 include nix.conf.extra
+trusted-users = $(whoami)
 EOF
 
 cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF

tests/lang.sh

@@ -2,9 +2,9 @@ source common.sh
 
 export TEST_VAR=foo # for eval-okay-getenv.nix
 
-nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
+nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello
-(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
+nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello
-nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
+expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grepQuiet Hello
 
 set +x
 

tests/linux-sandbox.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon"
+
 clearStore
 
 if ! canUseSandbox; then exit; fi

tests/local.mk

@@ -1,6 +1,10 @@
 check:
 	@echo "Warning: Nix has no 'make check'. Please install Nix and run 'make installcheck' instead."
 
+# whether to run the tests that assume that we have a local build of
+# Nix
+HAVE_LOCAL_NIX_BUILD ?= 1
+
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   gc.sh \
@@ -27,17 +31,23 @@ nix_tests = \
   brotli.sh \
   pure-eval.sh \
   check.sh \
-  plugins.sh \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
-  function-trace.sh
+  function-trace.sh \
+  test-infra.sh \
   # parallel.sh
 
+ifeq ($(HAVE_LOCAL_NIX_BUILD), 1)
+  nix_tests += plugins.sh
+endif
+
+tests/plugins.sh.test: tests/plugins/libplugintest.$(SO_EXT)
+
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 
 tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-test-deps += tests/common.sh tests/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh

tests/misc.sh

@@ -3,17 +3,17 @@ source common.sh
 # Tests miscellaneous commands.
 
 # Do all commands have help?
-#nix-env --help | grep -q install
+#nix-env --help | grepQuiet install
-#nix-store --help | grep -q realise
+#nix-store --help | grepQuiet realise
-#nix-instantiate --help | grep -q eval
+#nix-instantiate --help | grepQuiet eval
-#nix-hash --help | grep -q base32
+#nix-hash --help | grepQuiet base32
 
 # Can we ask for the version number?
 nix-env --version | grep "$version"
 
 # Usage errors.
-nix-env --foo 2>&1 | grep "no operation"
+expect 1 nix-env --foo 2>&1 | grep "no operation"
-nix-env -q --foo 2>&1 | grep "unknown flag"
+expect 1 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"
+expect 1 nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"

tests/multiple-outputs.sh

@@ -13,8 +13,8 @@ echo "evaluating c..."
 # outputs.
 drvPath=$(nix-instantiate multiple-outputs.nix -A c)
 #[ "$drvPath" = "$drvPath2" ]
-grep -q 'multiple-outputs-a.drv",\["first","second"\]' $drvPath
+grepQuiet 'multiple-outputs-a.drv",\["first","second"\]' $drvPath
-grep -q 'multiple-outputs-b.drv",\["out"\]' $drvPath
+grepQuiet 'multiple-outputs-b.drv",\["out"\]' $drvPath
 
 # While we're at it, test the ‘unsafeDiscardOutputDependency’ primop.
 outPath=$(nix-build multiple-outputs.nix -A d --no-out-link)
@@ -52,7 +52,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou
 
 # Delete one of the outputs and rebuild it.  This will cause a hash
 # rewrite.
-nix-store --delete $TEST_ROOT/result-second --ignore-liveness
+env -u NIX_REMOTE nix-store --delete $TEST_ROOT/result-second --ignore-liveness
 nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
 [ "$(cat $TEST_ROOT/result-second/file)" = "second" ]
 [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ]

tests/nar-access.sh

@@ -34,8 +34,8 @@ diff -u baz.cat-nar $storePath/foo/baz
 [[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]
 
 # Test missing files.
-nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
+expect 1 nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
-nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist'
+expect 1 nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist'
 
 # Test failure to dump.
 if nix-store --dump $storePath >/dev/full ; then

tests/nix-channel.sh

@@ -6,7 +6,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile
 
 # Test add/list/remove.
 nix-channel --add http://foo/bar xyzzy
-nix-channel --list | grep -q http://foo/bar
+nix-channel --list | grepQuiet http://foo/bar
 nix-channel --remove xyzzy
 
 [ -e $TEST_HOME/.nix-channels ]
@@ -31,8 +31,8 @@ nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml
 if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
-grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
+grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grepQuiet 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
 
 # Do an install.
 nix-env -i dependencies
@@ -50,8 +50,8 @@ nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml
 if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
-grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
+grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grepQuiet 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
 
 # Do an install.
 nix-env -i dependencies

tests/optimise-store.sh

@@ -26,7 +26,8 @@ if [ "$inode1" = "$inode3" ]; then
     exit 1
 fi
 
-nix-store --optimise
+# XXX: This should work through the daemon too
+NIX_REMOTE="" nix-store --optimise
 
 inode1="$(stat --format=%i $outPath1/foo)"
 inode3="$(stat --format=%i $outPath3/foo)"

tests/post-hook.sh

@@ -2,7 +2,12 @@ source common.sh
 
 clearStore
 
+rm -f $TEST_ROOT/result
+
 export REMOTE_STORE=$TEST_ROOT/remote_store
+echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf
+
+restartDaemon
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh

tests/referrers.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "uses some low-level store manipulations that aren’t available through the daemon"
+
 clearStore
 
 max=500

tests/remote-store.sh

@@ -4,14 +4,14 @@ clearStore
 
 startDaemon
 
-storeCleared=1 $SHELL ./user-envs.sh
+storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
+
+nix-store --gc --max-freed 1K
 
 nix-store --dump-db > $TEST_ROOT/d1
 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2
 cmp $TEST_ROOT/d1 $TEST_ROOT/d2
 
-nix-store --gc --max-freed 1K
-
 killDaemon
 
 user=$(whoami)

tests/repair.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--repair needs a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)

tests/search.sh

@@ -38,6 +38,6 @@ clearCache
 ## Search expressions
 
 # Check that empty search string matches all
-nix search|grep -q foo
+nix search|grepQuiet foo
-nix search|grep -q bar
+nix search|grepQuiet bar
-nix search|grep -q hello
+nix search|grepQuiet hello

tests/structured-attrs.sh

@@ -1,5 +1,9 @@
 source common.sh
 
+# 27ce722638 required some incompatible changes to the nix file, so skip this
+# tests for the newer versions
+requireDaemonOlderThan "2.4pre20210622"
+
 clearStore
 
 outPath=$(nix-build structured-attrs.nix --no-out-link)

tests/tarball.sh

@@ -13,7 +13,7 @@ cp config.nix dependencies.builder*.sh $tarroot/
 tarball=$TEST_ROOT/tarball.tar.xz
 (cd $TEST_ROOT && tar cf - tarball) | xz > $tarball
 
-nix-env -f file://$tarball -qa --out-path | grep -q dependencies
+nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies
 
 nix-build -o $TEST_ROOT/result file://$tarball
 

tests/test-infra.sh

@@ -0,0 +1,85 @@
+# Test the functions for testing themselves!
+# Also test some assumptions on how bash works that they rely on.
+source common.sh
+
+# `true` should exit with 0
+expect 0 true
+
+# `false` should exit with 1
+expect 1 false
+
+# `expect` will fail when we get it wrong
+expect 1 expect 0 false
+
+noisyTrue () {
+    echo YAY! >&2
+    true
+}
+
+noisyFalse () {
+    echo NAY! >&2
+    false
+}
+
+# These should redirect standard error to standard output
+expectStderr 0 noisyTrue | grepQuiet YAY
+expectStderr 1 noisyFalse | grepQuiet NAY
+
+# `set -o pipefile` is enabled
+
+pipefailure () {
+    # shellcheck disable=SC2216
+    true | false | true
+}
+expect 1 pipefailure
+unset pipefailure
+
+pipefailure () {
+    # shellcheck disable=SC2216
+    false | true | true
+}
+expect 1 pipefailure
+unset pipefailure
+
+commandSubstitutionPipeFailure () {
+    # shellcheck disable=SC2216
+    res=$(set -eu -o pipefail; false | true | echo 0)
+}
+expect 1 commandSubstitutionPipeFailure
+
+# `set -u` is enabled
+
+# note (...), making function use subshell, as unbound variable errors
+# in the outer shell are *rightly* not recoverable.
+useUnbound () (
+    set -eu
+    # shellcheck disable=SC2154
+    echo "$thisVariableIsNotBound"
+)
+expect 1 useUnbound
+
+# ! alone unfortunately negates `set -e`, but it works in functions:
+# shellcheck disable=SC2251
+! true
+funBang () {
+    ! true
+}
+expect 1 funBang
+unset funBang
+
+# `grep -v -q` is not what we want for exit codes, but `grepInverse` is
+# Avoid `grep -v -q`. The following line proves the point, and if it fails,
+# we'll know that `grep` had a breaking change or `-v -q` may not be portable.
+{ echo foo; echo bar; } | grep -v -q foo
+{ echo foo; echo bar; } | expect 1 grepInverse foo
+
+# `grepQuiet` is quiet
+res=$(set -eu -o pipefail; echo foo | grepQuiet foo | wc -c)
+(( res == 0 ))
+unset res
+
+# `greqQietInverse` is both
+{ echo foo; echo bar; } | expect 1 grepQuietInverse foo
+res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c)
+(( res == 0 ))
+unset res

tests/timeout.sh

@@ -2,18 +2,17 @@
 
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+needLocalStore "see #4813"
 
-set +e
+messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$?
-messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1)
-status=$?
-set -e
 
 if [ $status -ne 101 ]; then
     echo "error: 'nix-store' exited with '$status'; should have exited 101"
     exit 1
 fi
 
-if ! echo "$messages" | grep -q "timed out"; then
+if echo "$messages" | grepQuietInvert "timed out"; then
     echo "error: build may have failed for reasons other than timeout; output:"
     echo "$messages" >&2
     exit 1

tests/user-envs.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-if [ -z "$storeCleared" ]; then
+if [ -z "${storeCleared-}" ]; then
     clearStore
 fi
 
@@ -19,13 +19,13 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1
 [ -n "$outPath10" -a -n "$drvPath10" ]
 
 # Query descriptions.
-nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly
+nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly
 rm -rf $HOME/.nix-defexpr
 ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr
-nix-env -qa '*' --description | grep -q silly
+nix-env -qa '*' --description | grepQuiet silly
 
 # Query the system.
-nix-env -qa '*' --system | grep -q $system
+nix-env -qa '*' --system | grepQuiet $system
 
 # Install "foo-1.0".
 nix-env -i foo-1.0
@@ -33,19 +33,19 @@ nix-env -i foo-1.0
 # Query installed: should contain foo-1.0 now (which should be
 # executable).
 test "$(nix-env -q '*' | wc -l)" -eq 1
-nix-env -q '*' | grep -q foo-1.0
+nix-env -q '*' | grepQuiet foo-1.0
 test "$($profiles/test/bin/foo)" = "foo-1.0"
 
 # Test nix-env -qc to compare installed against available packages, and vice versa.
-nix-env -qc '*' | grep -q '< 2.0'
+nix-env -qc '*' | grepQuiet '< 2.0'
-nix-env -qac '*' | grep -q '> 1.0'
+nix-env -qac '*' | grepQuiet '> 1.0'
 
 # Test the -b flag to filter out source-only packages.
 [ "$(nix-env -qab | wc -l)" -eq 1 ]
 
 # Test the -s flag to get package status.
-nix-env -qas | grep -q 'IP-  foo-1.0'
+nix-env -qas | grepQuiet 'IP-  foo-1.0'
-nix-env -qas | grep -q -- '---  bar-0.1'
+nix-env -qas | grepQuiet -- '---  bar-0.1'
 
 # Disable foo.
 nix-env --set-flag active false foo
@@ -65,15 +65,15 @@ nix-env -i foo-2.0pre1
 
 # Query installed: should contain foo-2.0pre1 now.
 test "$(nix-env -q '*' | wc -l)" -eq 1
-nix-env -q '*' | grep -q foo-2.0pre1
+nix-env -q '*' | grepQuiet foo-2.0pre1
 test "$($profiles/test/bin/foo)" = "foo-2.0pre1"
 
 # Upgrade "foo": should install foo-2.0.
-NIX_PATH=nixpkgs=./user-envs.nix:$NIX_PATH nix-env -f '<nixpkgs>' -u foo
+NIX_PATH=nixpkgs=./user-envs.nix:${NIX_PATH-} nix-env -f '<nixpkgs>' -u foo
 
 # Query installed: should contain foo-2.0 now.
 test "$(nix-env -q '*' | wc -l)" -eq 1
-nix-env -q '*' | grep -q foo-2.0
+nix-env -q '*' | grepQuiet foo-2.0
 test "$($profiles/test/bin/foo)" = "foo-2.0"
 
 # Store the path of foo-2.0.
@@ -85,20 +85,20 @@ nix-env -i bar-0.1
 nix-env -e foo
 
 # Query installed: should only contain bar-0.1 now.
-if nix-env -q '*' | grep -q foo; then false; fi
+if nix-env -q '*' | grepQuiet foo; then false; fi
-nix-env -q '*' | grep -q bar
+nix-env -q '*' | grepQuiet bar
 
 # Rollback: should bring "foo" back.
 oldGen="$(nix-store -q --resolve $profiles/test)"
 nix-env --rollback
 [ "$(nix-store -q --resolve $profiles/test)" != "$oldGen" ]
-nix-env -q '*' | grep -q foo-2.0
+nix-env -q '*' | grepQuiet foo-2.0
-nix-env -q '*' | grep -q bar
+nix-env -q '*' | grepQuiet bar
 
 # Rollback again: should remove "bar".
 nix-env --rollback
-nix-env -q '*' | grep -q foo-2.0
+nix-env -q '*' | grepQuiet foo-2.0
-if nix-env -q '*' | grep -q bar; then false; fi
+if nix-env -q '*' | grepQuiet bar; then false; fi
 
 # Count generations.
 nix-env --list-generations
@@ -120,7 +120,7 @@ nix-env --switch-generation 7
 
 # Install foo-1.0, now using its store path.
 nix-env -i "$outPath10"
-nix-env -q '*' | grep -q foo-1.0
+nix-env -q '*' | grepQuiet foo-1.0
 nix-store -qR $profiles/test | grep "$outPath10"
 nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve $profiles/test)"
 [ "$(nix-store -q --deriver "$outPath10")" = $drvPath10 ]
@@ -128,12 +128,12 @@ nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve
 # Uninstall foo-1.0, using a symlink to its store path.
 ln -sfn $outPath10/bin/foo $TEST_ROOT/symlink
 nix-env -e $TEST_ROOT/symlink
-if nix-env -q '*' | grep -q foo; then false; fi
+if nix-env -q '*' | grepQuiet foo; then false; fi
-(! nix-store -qR $profiles/test | grep "$outPath10")
+nix-store -qR $profiles/test | grepInverse "$outPath10"
 
 # Install foo-1.0, now using a symlink to its store path.
 nix-env -i $TEST_ROOT/symlink
-nix-env -q '*' | grep -q foo
+nix-env -q '*' | grepQuiet foo
 
 # Delete all old generations.
 nix-env --delete-generations old
@@ -151,7 +151,7 @@ test "$(nix-env -q '*' | wc -l)" -eq 0
 # Installing "foo" should only install the newest foo.
 nix-env -i foo
 test "$(nix-env -q '*' | grep foo- | wc -l)" -eq 1
-nix-env -q '*' | grep -q foo-2.0
+nix-env -q '*' | grepQuiet foo-2.0
 
 # On the other hand, this should install both (and should fail due to
 # a collision).
@@ -162,8 +162,8 @@ nix-env -e '*'
 nix-env -e '*'
 nix-env -i '*'
 test "$(nix-env -q '*' | wc -l)" -eq 2
-nix-env -q '*' | grep -q foo-2.0
+nix-env -q '*' | grepQuiet foo-2.0
-nix-env -q '*' | grep -q bar-0.1.1
+nix-env -q '*' | grepQuiet bar-0.1.1
 
 # Test priorities: foo-0.1 has a lower priority than foo-1.0, so it
 # should be possible to install both without a collision.  Also test