Merge pull request #598 from nix-community/bats_testing

Switch to bats over pytest

.envrc

@@ -1,5 +1,9 @@
 # shellcheck shell=bash
 strict_env
 source ./direnvrc
-watch_file direnvrc ./*.nix
+
+watch_file direnvrc
+# shellcheck disable=SC2046
+watch_file $(find . -name "*.nix")
+
 use flake

direnvrc

@@ -37,10 +37,10 @@ _nix() {
 }
 
 _require_version() {
-  local cmd=$1 version=$2 required=$3
+  local cmd=$1 raw_version=$2 version=${2%%[^0-9.]*} required=$3
   if ! printf "%s\n" "$required" "$version" | LC_ALL=C sort -c -V 2>/dev/null; then
     _nix_direnv_error \
-      "minimum required $(basename "$cmd") version is $required (installed: $version)"
+      "minimum required $(basename "$cmd") version is $required (installed: $raw_version)"
     return 1
   fi
 }
@@ -52,7 +52,7 @@ _require_cmd_version() {
     return 1
   fi
   version=$($cmd --version)
-  [[ $version =~ ([0-9]+\.[0-9]+\.[0-9]+) ]]
+  [[ $version =~ ([0-9]+\.[0-9]+(\.[0-9]+)?) ]]
   _require_version "$cmd" "${BASH_REMATCH[1]}" "$required"
 }
 

flake.nix

@@ -33,19 +33,18 @@
             self',
             ...
           }:
+          let
+            nix-direnv = pkgs.callPackage ./default.nix { };
+            test_pkgs = pkgs.lib.callPackagesWith pkgs ./tests { inherit nix-direnv; };
+          in
           {
-            packages = {
-              nix-direnv = pkgs.callPackage ./default.nix { };
-              default = config.packages.nix-direnv;
-              test-runner-stable = pkgs.callPackage ./test-runner.nix { nixVersion = "stable"; };
-              test-runner-latest = pkgs.callPackage ./test-runner.nix { nixVersion = "latest"; };
+            packages = test_pkgs // {
+              inherit nix-direnv;
+              default = nix-direnv;
             };
 
             devShells.default = pkgs.callPackage ./shell.nix {
-              packages = [
-                config.treefmt.build.wrapper
-                pkgs.shellcheck
-              ];
+              treefmt = config.treefmt.build.wrapper;
             };
 
             checks =

shell.nix

@@ -1,14 +1,30 @@
 {
   pkgs ? import <nixpkgs> { },
-  packages ? [ ],
+  treefmt ? null,
+  nix-direnv ? (pkgs.callPackage ./default.nix { }),
+  test_pkgs ? (pkgs.lib.callPackagesWith pkgs ./tests { inherit nix-direnv; }),
 }:
-
-with pkgs;
-mkShell {
-  packages = packages ++ [
-    python3.pkgs.pytest
-    python3.pkgs.mypy
-    ruff
-    direnv
-  ];
+let
+  inherit (pkgs) lib;
+in
+pkgs.mkShell {
+  DIRENV_STDLIB = "${test_pkgs.direnv-stdlib}";
+  DIRENVRC = "${nix-direnv}/share/nix-direnv/direnvrc";
+  BATS_LIB_PATH = lib.strings.makeSearchPath "" (
+    with test_pkgs;
+    [
+      bats-support
+      bats-assert
+    ]
+  );
+  packages =
+    (builtins.attrValues {
+      inherit (pkgs)
+        bats
+        direnv
+        shellcheck
+        ;
+    })
+    ++ (builtins.attrValues (lib.attrsets.filterAttrs (name: _val: name != "direnv-stdlib") test_pkgs))
+    ++ lib.optionals (treefmt != null) [ treefmt ];
 }

test-runner.nix

@@ -1,24 +0,0 @@
-{
-  writeShellScriptBin,
-  direnv,
-  python3,
-  lib,
-  coreutils,
-  gnugrep,
-  nixVersions,
-  nixVersion,
-}:
-writeShellScriptBin "test-runner-${nixVersion}" ''
-  set -e
-  export PATH=${
-    lib.makeBinPath [
-      direnv
-      nixVersions.${nixVersion}
-      coreutils
-      gnugrep
-    ]
-  }
-
-  echo run unittest
-  ${lib.getExe' python3.pkgs.pytest "pytest"} .
-''

tests/conftest.py

@@ -1,18 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-pytest_plugins = [
-    "tests.direnv_project",
-    "tests.root",
-]
-
-
-@pytest.fixture(autouse=True)
-def _cleanenv(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    # so direnv doesn't touch $HOME
-    monkeypatch.setenv("HOME", str(tmp_path / "home"))
-    # so direnv allow state writes under tmp HOME
-    monkeypatch.delenv("XDG_DATA_HOME", raising=False)
-    # so direnv does not pick up user customization
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)

tests/default.nix

@@ -0,0 +1,54 @@
+{
+  bash,
+  bats,
+  callPackage,
+  coreutils,
+  direnv,
+  fetchurl,
+  findutils,
+  gnugrep,
+  lib,
+  nix-direnv,
+  nixVersions,
+  writeShellScriptBin,
+}:
+let
+  direnv-stdlib = fetchurl {
+    url = "https://raw.githubusercontent.com/direnv/direnv/refs/tags/v2.37.1/stdlib.sh";
+    hash = "sha256-MMM04OXhqS/rRSuv8uh7CD70Z7CaGT63EtL/3LC08qM=";
+  };
+  bats-support = callPackage ./nix/bats-support.nix { };
+  bats-assert = callPackage ./nix/bats-assert.nix { };
+  mkTestRunner =
+    nixVersion:
+    writeShellScriptBin "test-runner-${nixVersion}" ''
+      set -e
+      export PATH=${
+        lib.makeBinPath [
+          bash
+          direnv
+          nixVersions.${nixVersion}
+          coreutils
+          findutils
+          gnugrep
+        ]
+      }
+      export DIRENV_STDLIB=${direnv-stdlib}
+      export DIRENVRC="${nix-direnv}/share/nix-direnv/direnvrc"
+      export BATS_LIB_PATH="${bats-support}:${bats-assert}"
+
+      echo run unittest
+      ${lib.getExe' bats "bats"} -x --verbose-run tests/
+    '';
+  test-runner-stable = mkTestRunner "stable";
+  test-runner-latest = mkTestRunner "latest";
+in
+{
+  inherit
+    bats-support
+    bats-assert
+    direnv-stdlib
+    test-runner-stable
+    test-runner-latest
+    ;
+}

tests/direnv_project.py

@@ -1,48 +0,0 @@
-import shutil
-import textwrap
-from collections.abc import Iterator
-from dataclasses import dataclass
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-import pytest
-
-from .procs import run
-
-
-@dataclass
-class DirenvProject:
-    directory: Path
-    nix_direnv: Path
-
-    @property
-    def envrc(self) -> Path:
-        return self.directory / ".envrc"
-
-    def setup_envrc(self, content: str, strict_env: bool) -> None:
-        text = textwrap.dedent(
-            f"""
-        {"strict_env" if strict_env else ""}
-        source {self.nix_direnv}
-        {content}
-        """
-        )
-        self.envrc.write_text(text)
-        run(["direnv", "allow"], cwd=self.directory)
-
-
-@pytest.fixture
-def direnv_project(test_root: Path, project_root: Path) -> Iterator[DirenvProject]:
-    """
-    Setups a direnv test project
-    """
-    with TemporaryDirectory() as _dir:
-        directory = Path(_dir) / "proj"
-        shutil.copytree(test_root / "testenv", directory)
-        nix_direnv = project_root / "direnvrc"
-
-        c = DirenvProject(Path(directory), nix_direnv)
-        try:
-            yield c
-        finally:
-            pass

tests/nix/bats-assert.nix

@@ -0,0 +1,23 @@
+{ fetchFromGitHub, stdenv }:
+stdenv.mkDerivation {
+  name = "bats-assert";
+  version = "2.1.0+";
+  src = fetchFromGitHub {
+    owner = "bats-core";
+    repo = "bats-assert";
+    rev = "912a98804efd34f24d5eae1bf97ee622ca770e9"; # master 8/7/2025
+    hash = "sha256-gp52V4mAiT+Lod2rvEMLhi0Y7AdQQTFCHcNgb8JEKXE=";
+  };
+
+  dontBuild = true;
+  installPhase = ''
+
+    # This looks funny
+    # but they mean that you can use bats' built-in `bats_load_library` easily
+    # when setting $BATS_LIB_PATH to the string of the derivation.
+
+    mkdir -p $out/bats-assert;
+    cp -r src $out/bats-assert/
+    cp load.bash $out/bats-assert
+  '';
+}

tests/nix/bats-support.nix

@@ -0,0 +1,24 @@
+{ stdenv, fetchFromGitHub }:
+
+stdenv.mkDerivation {
+  name = "bats-support";
+  version = "3.0+";
+  src = fetchFromGitHub {
+    owner = "bats-core";
+    repo = "bats-support";
+    rev = "0ad082d4590108684c68975ca517a90459f05cd0";
+    hash = "sha256-hkPAn12gQudboL9pDpQZhtaMhqyyj885tti4Gx/aun4=";
+  };
+
+  dontBuild = true;
+  installPhase = ''
+
+    # This looks funny
+    # but they mean that you can use bats' built-in `bats_load_library` easily
+    # when setting $BATS_LIB_PATH to the string of the derivation.
+
+    mkdir -p $out/bats-support;
+    cp -r src $out/bats-support/
+    cp load.bash $out/bats-support/
+  '';
+}

tests/procs.py

@@ -1,27 +0,0 @@
-import logging
-import shlex
-import subprocess
-from pathlib import Path
-from typing import IO, Any
-
-_FILE = None | int | IO[Any]
-_DIR = None | Path | str
-
-log = logging.getLogger(__name__)
-
-
-def run(
-    cmd: list[str],
-    text: bool = True,
-    check: bool = True,
-    cwd: _DIR = None,
-    stderr: _FILE = None,
-    stdout: _FILE = None,
-    env: dict[str, str] | None = None,
-) -> subprocess.CompletedProcess:
-    if cwd is not None:
-        log.debug(f"cd {cwd}")
-    log.debug(f"$ {shlex.join(cmd)}")
-    return subprocess.run(
-        cmd, text=text, check=check, cwd=cwd, stderr=stderr, stdout=stdout, env=env
-    )

tests/root.py

@@ -1,22 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-TEST_ROOT = Path(__file__).parent.resolve()
-PROJECT_ROOT = TEST_ROOT.parent
-
-
-@pytest.fixture
-def test_root() -> Path:
-    """
-    Root directory of the tests
-    """
-    return TEST_ROOT
-
-
-@pytest.fixture
-def project_root() -> Path:
-    """
-    Root directory of the tests
-    """
-    return PROJECT_ROOT

tests/test_gc.bats

@@ -0,0 +1,73 @@
+# -*- mode: bash-ts -*-
+
+# test initialization ====================
+function setup {
+  load "util"
+
+  _common_setup
+}
+
+function teardown {
+  _common_teardown
+}
+
+# helpers =================================
+function assert_run_output {
+  run_in_direnv hello
+  assert_output -p "Hello, world"
+  assert_stderr -p "Executing shellHook"
+}
+
+function assert_gcroot {
+  profile_path=$(find "$TESTDIR/.direnv" -type l | head -n 1)
+  run bats_pipe find /nix/var/nix/gcroots/auto/ -type l -printf "%l\n" \| grep -q "$profile_path"
+  assert_success
+}
+
+function assert_use_nix_layout_dir_shape {
+  paths=("$TESTDIR"/.direnv/*)
+  chomped_paths=("${paths[@]#$TESTDIR/.direnv/}")
+  assert_equal "${#chomped_paths[@]}" "3"
+  assert_regex "$(printf "%s " "${chomped_paths[@]}")" "bin nix-profile.+ nix-profile-.+\.rc"
+}
+
+function assert_use_flake_layout_dir_shape {
+  paths=("$TESTDIR"/.direnv/flake-inputs/*)
+  chomped_inputs_paths=("${paths[@]#$TESTDIR/.direnv/flake-inputs/}")
+  # four inputs, so four "...-source" outputs
+  assert_regex "$(printf "%s " "${chomped_inputs_paths[@]}")" "(.+-source[ ]?){4}"
+
+  paths=("$TESTDIR"/.direnv/*)
+  chomped_paths=("${paths[@]#$TESTDIR/.direnv/}")
+  assert_equal "${#chomped_paths[@]}" "4"
+  assert_regex "$(printf "%s " "${chomped_paths[@]}")" "bin flake-inputs flake-profile-.+ flake-profile-.+\.rc"
+}
+
+# tests ===================================
+function use_nix_no_strict { # @test
+  write_envrc "use nix"
+  assert_run_output
+  assert_gcroot
+  assert_use_nix_layout_dir_shape
+}
+
+function use_nix_strict { # @test
+  write_envrc "strict_env\nuse nix"
+  assert_run_output
+  assert_gcroot
+  assert_use_nix_layout_dir_shape
+}
+
+function use_flake_no_strict { # @test
+  write_envrc "use flake"
+  assert_run_output
+  assert_gcroot
+  assert_use_flake_layout_dir_shape
+}
+
+function use_flake_strict { # @test
+  write_envrc "strict_env\nuse flake"
+  assert_run_output
+  assert_gcroot
+  assert_use_flake_layout_dir_shape
+}

tests/test_gc.py

@@ -1,98 +0,0 @@
-import logging
-import subprocess
-import sys
-import unittest
-
-import pytest
-
-from .direnv_project import DirenvProject
-from .procs import run
-
-log = logging.getLogger(__name__)
-
-
-def common_test(direnv_project: DirenvProject) -> None:
-    run(["nix-collect-garbage"])
-
-    testenv = str(direnv_project.directory)
-
-    out1 = run(
-        ["direnv", "exec", testenv, "hello"],
-        stderr=subprocess.PIPE,
-        check=False,
-        cwd=direnv_project.directory,
-    )
-    sys.stderr.write(out1.stderr)
-    assert out1.returncode == 0
-    assert "Renewed cache" in out1.stderr
-    assert "Executing shellHook." in out1.stderr
-
-    run(["nix-collect-garbage"])
-
-    out2 = run(
-        ["direnv", "exec", testenv, "hello"],
-        stderr=subprocess.PIPE,
-        check=False,
-        cwd=direnv_project.directory,
-    )
-    sys.stderr.write(out2.stderr)
-    assert out2.returncode == 0
-    assert "Using cached dev shell" in out2.stderr
-    assert "Executing shellHook." in out2.stderr
-
-
-def common_test_clean(direnv_project: DirenvProject) -> None:
-    testenv = str(direnv_project.directory)
-
-    out3 = run(
-        ["direnv", "exec", testenv, "hello"],
-        stderr=subprocess.PIPE,
-        check=False,
-        cwd=direnv_project.directory,
-    )
-    sys.stderr.write(out3.stderr)
-
-    files = [
-        path
-        for path in (direnv_project.directory / ".direnv").iterdir()
-        if path.is_file()
-    ]
-    rcs = [f for f in files if f.match("*.rc")]
-    profiles = [f for f in files if not f.match("*.rc")]
-    if len(rcs) != 1 or len(profiles) != 1:
-        log.debug(files)
-    assert len(rcs) == 1
-    assert len(profiles) == 1
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_use_nix(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use nix", strict_env=strict_env)
-    common_test(direnv_project)
-
-    direnv_project.setup_envrc(
-        "use nix --argstr shellHook 'echo Executing hijacked shellHook.'",
-        strict_env=strict_env,
-    )
-    common_test_clean(direnv_project)
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_use_flake(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use flake", strict_env=strict_env)
-    common_test(direnv_project)
-    inputs = list((direnv_project.directory / ".direnv/flake-inputs").iterdir())
-    # should only contain our flake-utils flake
-    if len(inputs) != 4:
-        run(["nix", "flake", "archive", "--json"], cwd=direnv_project.directory)
-        log.debug(inputs)
-    assert len(inputs) == 4
-    for symlink in inputs:
-        assert symlink.is_dir()
-
-    direnv_project.setup_envrc("use flake --impure", strict_env=strict_env)
-    common_test_clean(direnv_project)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_use_nix.bats

@@ -0,0 +1,51 @@
+# -*- mode: bash-ts -*-
+
+function setup {
+  load "util"
+
+  _common_setup
+}
+
+function teardown {
+  _common_teardown
+}
+
+function use_nix_attrs_strict { # @test
+  write_envrc "strict_env\nuse nix -A subshell"
+  # shellcheck disable=SC2016
+  run_in_direnv 'echo "subshell: $THIS_IS_A_SUBSHELL"'
+  assert_output -e "subshell: OK$"
+}
+
+function use_nix_attrs_no_strict { # @test
+  write_envrc "use nix -A subshell"
+  # shellcheck disable=SC2016
+  run_in_direnv 'echo "subshell: $THIS_IS_A_SUBSHELL"'
+  assert_output -e "subshell: OK$"
+}
+
+function use_nix_no_nix_path_strict { # @test
+  unset NIX_PATH
+  write_envrc "strict_env\nuse nix --argstr someArg OK"
+  # shellcheck disable=SC2016
+  run_in_direnv 'echo "someArg: $SHOULD_BE_SET"'
+  assert_output -e "someArg: OK$"
+}
+
+function use_nix_no_nix_path_no_strict { # @test
+  unset NIX_PATH
+  write_envrc "use nix --argstr someArg OK"
+  # shellcheck disable=SC2016
+  run_in_direnv 'echo "someArg: $SHOULD_BE_SET"'
+  assert_output -e "someArg: OK$"
+}
+
+function use_nix_no_files { # @test
+  write_envrc "use nix -p hello"
+  (
+    cd "$TESTDIR" || exit 1
+    run --separate-stderr direnv status
+    assert_success
+    refute_output -p 'Loaded watch: "."'
+  )
+}

tests/test_use_nix.py

@@ -1,71 +0,0 @@
-import logging
-import os
-import shlex
-import subprocess
-import sys
-import unittest
-
-import pytest
-
-from .direnv_project import DirenvProject
-from .procs import run
-
-log = logging.getLogger(__name__)
-
-
-def direnv_exec(
-    direnv_project: DirenvProject, cmd: str, env: dict[str, str] | None = None
-) -> None:
-    args = ["direnv", "exec", str(direnv_project.directory), "sh", "-c", cmd]
-    log.debug(f"$ {shlex.join(args)}")
-    out = run(
-        args,
-        stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        check=False,
-        cwd=direnv_project.directory,
-        env=env,
-    )
-    sys.stdout.write(out.stdout)
-    sys.stderr.write(out.stderr)
-    assert out.returncode == 0
-    assert out.stdout == "OK\n"
-    assert "Renewed cache" in out.stderr
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_attrs(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use nix -A subshell", strict_env=strict_env)
-    direnv_exec(direnv_project, "echo $THIS_IS_A_SUBSHELL")
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_no_nix_path(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use nix --argstr someArg OK", strict_env=strict_env)
-    env = os.environ.copy()
-    del env["NIX_PATH"]
-    direnv_exec(direnv_project, "echo $SHOULD_BE_SET", env=env)
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_args(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use nix --argstr someArg OK", strict_env=strict_env)
-    direnv_exec(direnv_project, "echo $SHOULD_BE_SET")
-
-
-@pytest.mark.parametrize("strict_env", [False, True])
-def test_no_files(direnv_project: DirenvProject, strict_env: bool) -> None:
-    direnv_project.setup_envrc("use nix -p hello", strict_env=strict_env)
-    out = run(
-        ["direnv", "status"],
-        stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        check=False,
-        cwd=direnv_project.directory,
-    )
-    assert out.returncode == 0
-    assert 'Loaded watch: "."' not in out.stdout
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_versions.bats

@@ -0,0 +1,32 @@
+# -*- mode: bash-ts -*-
+
+function setup {
+  bats_load_library bats-support
+  bats_load_library bats-assert
+
+  load "$DIRENV_STDLIB"
+  load "$DIRENVRC"
+}
+
+function _require_version_with_valid_versions { # @test
+  # args: cmd version minimum_required
+  run _require_version "test-cmd" "2.5" "2.4"
+  assert_success
+  run _require_version "test-cmd" "2.5" "2.4.1"
+  assert_success
+  run _require_version "test-cmd" "2.4.1" "2.4"
+  assert_success
+  run _require_version "test-cmd" "2.4" "2.4.1"
+  assert_failure
+  run _require_version "test-cmd" "2.31pre20250712_b1245123" "2.4"
+  assert_success
+}
+
+function _require_cmd_version_with_valid_versions { # @test
+  run _require_cmd_version "bash" "1.0"
+  assert_success
+  run _require_cmd_version "bash" "100.0"
+  assert_failure
+  run _require_cmd_version "bash" "1.2.3"
+  assert_success
+}

tests/util.bash

@@ -0,0 +1,38 @@
+function _common_setup {
+  shopt -s globstar
+  bats_require_minimum_version 1.5.0
+  bats_load_library bats-support
+  bats_load_library bats-assert
+
+  TESTDIR=
+  TESTDIR=$(mktemp -d -t nix-direnv.XXXXXX)
+  export TESTDIR
+  export DIRENV_LOG_FORMAT="direnv: %s"
+
+  # Set up nix to be able to find your user's nix.conf if run locally
+  export NIX_USER_CONF_FILES="$HOME/.config/nix/nix.conf"
+
+  export HOME=$TESTDIR/home
+  unset XDG_DATA_HOME
+  unset XDG_CONFIG_HOME
+
+  cp "$BATS_TEST_DIRNAME"/testenv/* "$TESTDIR/"
+}
+
+function _common_teardown {
+  rm -rf "$TESTDIR"
+}
+
+function write_envrc {
+  echo "source $DIRENVRC" >"$TESTDIR/.envrc"
+  echo -e "\n$*" >>"$TESTDIR/.envrc"
+  direnv allow "$TESTDIR"
+}
+
+function run_in_direnv {
+  run --separate-stderr direnv exec "$TESTDIR" sh -c "$@"
+  assert_success
+  run direnv exec "$TESTDIR" sh -c "$@"
+  assert_success
+  assert_stderr -p "Renewed cache"
+}

treefmt.nix

@@ -12,9 +12,6 @@
         programs = {
           deadnix.enable = true;
           deno.enable = true;
-          mypy.enable = true;
-          ruff.check = true;
-          ruff.format = true;
           nixfmt.enable = true;
           nixfmt.package = pkgs.nixfmt-rfc-style;
           shellcheck.enable = true;
@@ -24,8 +21,16 @@
         };
 
         settings.formatter = {
-          shellcheck.includes = [ "direnvrc" ];
-          shfmt.includes = [ "direnvrc" ];
+          shellcheck.includes = [
+            "direnvrc"
+            "tests/*.bash"
+            "tests/*.bats"
+          ];
+          shfmt.includes = [
+            "direnvrc"
+            "tests/*.bash"
+            "tests/*.bats"
+          ];
         };
       };
     };