1
0
Fork 0
mirror of https://github.com/nix-community/home-manager.git synced 2025-11-08 11:36:05 +01:00

rclone: move activation script to systemd service

Fixes #7577
This lets us better express activation order dependencies on secret
provisioners that run as systemd services
This commit is contained in:
Jess 2025-08-18 03:01:11 +12:00 committed by Austin Horstman
parent 56b8749987
commit 3001400e9f
13 changed files with 531 additions and 82 deletions

View file

@ -10,9 +10,25 @@ let
cfg = config.programs.rclone;
iniFormat = pkgs.formats.ini { };
replaceSlashes = builtins.replaceStrings [ "/" ] [ "." ];
isUsingSecretProvisioner = name: config ? "${name}" && config."${name}".secrets != { };
in
{
imports = [
(lib.mkRemovedOptionModule [ "programs" "rclone" "writeAfter" ] ''
The writeAfter option has been removed because rclone configuration is now handled by a
systemd service instead of an activation script.
For most users, no manual configuration is needed as the following secret provisioners are
automatically detected:
- agenix users: automatically uses agenix.service
- sops-nix users: automatically uses sops-nix.service
If you need custom service dependencies, use the requiresUnit option instead:
programs.rclone.requiresUnit = "your-service-name.service";
'')
];
options = {
programs.rclone = {
enable = lib.mkEnableOption "rclone";
@ -76,11 +92,9 @@ in
must be provided as file paths to the secrets, which will be read at activation
time.
Note: If using secret management solutions like agenix or sops-nix with
home-manager, you need to ensure their services are activated before switching
to this home-manager generation. Consider setting
{option}`systemd.user.startServices` to `"sd-switch"` for automatic service
startup.
These values are expanded in a shell context within a systemd service, so
you can use bash features like command substitution or variable expansion
(e.g. "''${XDG_RUNTIME_DIR}" as used by agenix).
'';
example = lib.literalExpression ''
{
@ -197,99 +211,175 @@ in
}'';
};
writeAfter = lib.mkOption {
type = lib.types.str;
default = "reloadSystemd";
requiresUnit = lib.mkOption {
type = with lib.types; nullOr str;
default =
lib.foldlAttrs
(
acc: prov: svc:
if isUsingSecretProvisioner prov then svc else acc
)
null
{
"sops" = "sops-nix.service";
"age" = "agenix.service";
};
example = "agenix.service";
description = ''
Controls when the rclone configuration is written during Home Manager activation.
You should not need to change this unless you have very specific activation order
requirements.
The name of a systemd user service that must complete before the rclone
configuration file is written.
This is typically used when secrets are managed by an external provisioner
whose service must run before the secrets are accessible.
When using sops-nix or agenix, this value is set automatically to
sops-nix.service or agenix.service, respectively. Set this manually if you
use a different secret provisioner.
'';
};
};
};
config = lib.mkIf cfg.enable {
home = {
packages = [ cfg.package ];
activation.createRcloneConfig =
config =
let
rcloneConfigService =
let
safeConfig = lib.pipe cfg.remotes [
(lib.mapAttrs (_: v: v.config))
(iniFormat.generate "rclone.conf@pre-secrets")
];
# https://github.com/rclone/rclone/issues/8190
injectSecret =
remote:
lib.mapAttrsToList (secret: secretFile: ''
${lib.getExe cfg.package} config update \
${remote.name} config_refresh_token=false \
${secret} "$(cat ${secretFile})" \
--quiet --non-interactive > /dev/null
if ! cat "${secretFile}"; then
echo "Secret \"${secretFile}\" not found"
cleanup
fi
if ! ${lib.getExe cfg.package} config update \
${remote.name} config_refresh_token=false \
${secret} "$(cat "${secretFile}")" \
--non-interactive; then
echo "Failed to inject secret \"${secretFile}\""
cleanup
fi
'') remote.value.secrets or { };
injectAllSecrets = lib.concatMap injectSecret (lib.mapAttrsToList lib.nameValuePair cfg.remotes);
rcloneConfigPath = "${config.xdg.configHome}/rclone/rclone.conf";
in
lib.mkIf (cfg.remotes != { }) (
lib.hm.dag.entryAfter [ "writeBoundary" cfg.writeAfter ] ''
run install $VERBOSE_ARG -D -m600 ${safeConfig} "${config.xdg.configHome}/rclone/rclone.conf"
${lib.concatLines injectAllSecrets}
''
);
};
lib.mkIf (cfg.remotes != { }) {
rclone-config = {
Unit = lib.mkMerge [
{
Description = "Install rclone configuration to ${rcloneConfigPath}";
}
systemd.user.services = lib.listToAttrs (
lib.concatMap
(
{ name, value }:
let
remote-name = name;
remote = value;
in
lib.concatMap (
(lib.optionalAttrs (cfg.requiresUnit != null) {
Requires = [ cfg.requiresUnit ];
After = [ cfg.requiresUnit ];
})
];
Service = {
Type = "oneshot";
ExecStart = lib.getExe (
pkgs.writeShellApplication {
name = "rclone-config";
runtimeInputs = [
pkgs.coreutils
];
text = ''
configPath="${rcloneConfigPath}"
configName="$(basename $configPath)"
savedConfigPath="$(dirname $configPath)"/."$configName".orig
cleanup() {
echo "Failed to render config."
if [ -f "$savedConfigPath" ]; then
cp -v "$savedConfigPath" "${rcloneConfigPath}"
fi
exit 1
}
trap cleanup SIGINT
if [ -f "${rcloneConfigPath}" ]; then
cp -v "${rcloneConfigPath}" "$savedConfigPath"
fi
install -v -D -m600 "${safeConfig}" "${rcloneConfigPath}"
${lib.concatLines injectAllSecrets}
'';
}
);
Restart = "on-abnormal";
};
Install.WantedBy = [ "default.target" ];
};
};
mountServices = lib.listToAttrs (
lib.concatMap
(
{ name, value }:
let
mount-path = name;
mount = value;
remote-name = name;
remote = value;
in
[
(lib.nameValuePair "rclone-mount:${replaceSlashes mount-path}@${remote-name}" {
Unit = {
Description = "Rclone FUSE daemon for ${remote-name}:${mount-path}";
};
lib.concatMap (
{ name, value }:
let
mount-path = name;
mount = value;
in
[
(lib.nameValuePair "rclone-mount:${replaceSlashes mount-path}@${remote-name}" {
Unit = {
Description = "Rclone FUSE daemon for ${remote-name}:${mount-path}";
};
Service = {
Environment = [
# fusermount/fusermount3
"PATH=/run/wrappers/bin"
];
ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${mount.mountPoint}";
ExecStart = lib.concatStringsSep " " [
(lib.getExe cfg.package)
"mount"
"-vv"
(lib.cli.toGNUCommandLineShell { } mount.options)
"${remote-name}:${mount-path}"
"${mount.mountPoint}"
];
Restart = "on-failure";
};
Service = {
Environment = [
# fusermount/fusermount3
"PATH=/run/wrappers/bin"
];
ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${mount.mountPoint}";
ExecStart = lib.concatStringsSep " " [
(lib.getExe cfg.package)
"mount"
"-vv"
(lib.cli.toGNUCommandLineShell { } mount.options)
"${remote-name}:${mount-path}"
"${mount.mountPoint}"
];
Restart = "on-failure";
};
Install.WantedBy = [ "default.target" ];
})
Install.WantedBy = [ "default.target" ];
})
]
) (lib.attrsToList remote.mounts)
)
(
lib.pipe cfg.remotes [
lib.attrsToList
(lib.filter (rem: rem.value ? mounts))
]
) (lib.attrsToList remote.mounts)
)
(
lib.pipe cfg.remotes [
lib.attrsToList
(lib.filter (rem: rem.value ? mounts))
]
)
);
};
)
);
in
lib.mkIf cfg.enable {
home.packages = [ cfg.package ];
systemd.user.services = lib.mkMerge [
rcloneConfigService
mountServices
];
};
meta.maintainers = with lib.maintainers; [ jess ];
}

View file

@ -21,6 +21,8 @@ let
nixos-basics = runTest ./nixos/basics.nix;
nixos-legacy-profile-management = runTest ./nixos/legacy-profile-management.nix;
rclone = runTest ./standalone/rclone;
rclone-sops-nix = runTest ./standalone/rclone/sops-nix.nix;
rclone-agenix = runTest ./standalone/rclone/agenix.nix;
restic = runTest ./standalone/restic.nix;
standalone-flake-basics = runTest ./standalone/flake-basics.nix;
standalone-specialisation = runTest ./standalone/specialisation.nix;

View file

@ -0,0 +1,104 @@
{ pkgs, lib, ... }:
let
commit = "9edb1787864c4f59ae5074ad498b6272b3ec308d";
sshKeys = import "${pkgs.path}/nixos/tests/ssh-keys.nix" pkgs;
agenix = pkgs.fetchzip {
url = "https://github.com/ryantm/agenix/archive/${commit}.tar.gz";
hash = "sha256-NA/FT2hVhKDftbHSwVnoRTFhes62+7dxZbxj5Gxvghs=";
};
passwordFile = pkgs.writeText "password" "aliceiscool2004";
passwordEnc = pkgs.runCommand "password-enc" { } ''
${lib.getExe pkgs.age} --encrypt \
--recipient "${sshKeys.snakeOilEd25519PublicKey}" \
--output $out \
${passwordFile}
'';
in
{
name = "rclone-agenix";
nodes.machine = {
imports = [ ../../../../nixos ];
virtualisation.memorySize = 2048;
users.users.alice = {
isNormalUser = true;
description = "Alice Foobar";
password = "foobar";
uid = 1000;
};
home-manager.users.alice =
{ config, ... }:
{
imports = [ "${agenix}/modules/age-home.nix" ];
home = {
username = "alice";
homeDirectory = "/home/alice";
stateVersion = "24.05"; # Please read the comment before changing.
};
age = {
identityPaths = [ "${sshKeys.snakeOilEd25519PrivateKey}" ];
secrets.password.file = "${passwordEnc}";
};
programs.rclone = {
enable = true;
remotes = {
alices-encrypted-files = {
config = {
type = "crypt";
remote = "/home/alice/enc-files";
};
secrets.password = config.age.secrets.password.path;
};
};
};
};
};
testScript = ''
def alice_cmd(cmd):
return f"su -l alice --shell /bin/sh -c $'export XDG_RUNTIME_DIR=/run/user/$UID ; {cmd}'"
def succeed_as_alice(*cmds, box=machine):
return box.succeed(*map(alice_cmd,cmds))
def assert_list(cmd, expected_list, actual):
assert all([x in actual for x in expected_list]), \
f"""Expected {cmd} to contain \
[{" and ".join([x for x in expected_list if x not in actual])}], but got {actual}"""
start_all()
machine.wait_for_unit("network.target")
machine.wait_for_unit("multi-user.target")
machine.wait_until_tty_matches("1", "login: ")
machine.send_chars("alice\n")
machine.wait_until_tty_matches("1", "Password: ")
machine.send_chars("foobar\n")
machine.wait_until_tty_matches("1", "alice\\@machine")
with subtest("Agenix activation ordering works correctly"):
# wait for rclone-config.service
machine.wait_until_succeeds("grep password /home/alice/.config/rclone/rclone.conf")
actual = succeed_as_alice("cat /home/alice/.config/rclone/rclone.conf")
assert_list("/home/alice/.config/rclone/rclone.conf", [
"[alices-encrypted-files]",
"remote = /home/alice/enc-files",
"type = crypt",
"password = "
], actual)
hidden_password = actual.strip().split("password = ")[1]
password = succeed_as_alice(f"rclone reveal {hidden_password}")
assert "aliceiscool2004" in password, \
f"Failed to decrypt password. Instead of aliceiscool2004, we got {password}"
'';
}

View file

@ -0,0 +1,64 @@
{ pkgs, ... }:
let
mkBrokenModule =
passPath:
pkgs.writeText "atomic-broken-module" ''
{
programs.rclone.remotes = {
alices-broken-test-remote = {
config = {
type = "smb";
host = "smb.alice.com";
user = "alice";
port = 1234;
};
secrets.pass = "${passPath}";
};
};
}
'';
moduleNoSuchFileDir = mkBrokenModule "/this/path/does/not/exist";
moduleSecretWithNewlines = mkBrokenModule (
pkgs.writeText "newline-secret" "\ra\n secret\nwith\r\nnewlines"
);
workingRemote = pkgs.writeText "atomic-working-remote" ''
[alices-working-remote]
host=backup-server
key_file=/key/path/foo
type=sftp
user=alice
'';
in
{
script = ''
# Test we dont overwrite a working config with a broken/partial one, after and error occurs.
with subtest("Writing the config is atomic through errors (no such file or directory)"):
succeed_as_alice("install -m644 ${moduleNoSuchFileDir} /home/alice/.config/home-manager/test-remote.nix")
succeed_as_alice("install -m644 -D ${workingRemote} /home/alice/.config/rclone/rclone.conf")
actual = succeed_as_alice("home-manager switch")
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${workingRemote} /home/alice/.config/rclone/rclone.conf")
exit_status = machine.get_unit_property("rclone-config.service", "Result", "alice")
assert "success" not in exit_status, "rclone-config.service unexpectedly ran successfully"
with subtest("Writing the config is atomic through errors (secret with newlines)"):
succeed_as_alice("install -m644 ${moduleSecretWithNewlines} /home/alice/.config/home-manager/test-remote.nix")
actual = succeed_as_alice("home-manager switch")
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${workingRemote} /home/alice/.config/rclone/rclone.conf")
exit_status = machine.get_unit_property("rclone-config.service", "Result", "alice")
assert "success" not in exit_status, "rclone-config.service unexpectedly ran successfully"
'';
}

View file

@ -24,6 +24,9 @@ in
./secrets-with-whitespace.nix
./no-type.nix
./mount.nix
./shell.nix
./atomic.nix
./write-after.nix
];
options.script = lib.mkOption {

View file

@ -61,8 +61,8 @@ in
)
actual = succeed_as_alice("home-manager switch")
expected = "Activating createRcloneConfig"
assert expected in actual, \
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
# remote -> machine

View file

@ -27,8 +27,8 @@ in
succeed_as_alice("install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix")
actual = succeed_as_alice("home-manager switch")
expected = "Activating createRcloneConfig"
assert expected in actual, \
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${expected} /home/alice/.config/rclone/rclone.conf")

View file

@ -18,9 +18,9 @@ in
succeed_as_alice("install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix")
actual = fail_as_alice("home-manager switch")
expected = "Activating createRcloneConfig"
expected = "rclone-config.service"
assert expected not in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
f"expected home-manager switch to not contain {expected}, but got {actual}"
expected = "An attribute set containing a remote type and options."
assert expected not in actual, \

View file

@ -30,8 +30,8 @@ in
succeed_as_alice("install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix")
actual = succeed_as_alice("home-manager switch")
expected = "Activating createRcloneConfig"
assert expected in actual, \
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${expected} /home/alice/.config/rclone/rclone.conf")

View file

@ -0,0 +1,57 @@
{ pkgs, config, ... }:
let
mkHttpModule =
httpHeadersPath:
pkgs.writeText "shell-module" ''
{
programs.rclone.remotes = {
alices-remote-with-shell-vars = {
config = {
type = "http";
url = "files.alice.com";
};
secrets.http-headers = "${httpHeadersPath}";
};
};
}
'';
expected = pkgs.writeText "shell-expected" ''
[alices-remote-with-shell-vars]
type = http
url = files.alice.com
http-headers = Cookie,secret_password=aliceiscool
'';
xdgRuntimeDir = "/run/user/${builtins.toString config.nodes.machine.users.users.alice.uid}";
httpHeadersSecret = pkgs.writeText "http-headers" "Cookie,secret_password=aliceiscool";
shellVar = mkHttpModule "\\\${XDG_RUNTIME_DIR}/http-headers";
shellCmd = mkHttpModule "$(printf '${xdgRuntimeDir}')/http-headers";
in
{
script = ''
succeed_as_alice("install -m644 ${httpHeadersSecret} ${xdgRuntimeDir}/http-headers")
def test_bash_expansion(module):
succeed_as_alice(f"install -m644 {module} /home/alice/.config/home-manager/test-remote.nix")
actual = succeed_as_alice("home-manager switch")
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${expected} /home/alice/.config/rclone/rclone.conf")
with subtest("Generate with shell variable in secrets"):
test_bash_expansion("${shellVar}")
# cleanup
succeed_as_alice("rm /home/alice/.config/rclone/rclone.conf")
succeed_as_alice("rm /home/alice/.config/home-manager/test-remote.nix")
with subtest("Generate with shell cmd in secrets"):
test_bash_expansion("${shellCmd}")
'';
}

View file

@ -0,0 +1,106 @@
{ pkgs, lib, ... }:
let
commit = "3223c7a92724b5d804e9988c6b447a0d09017d48";
agePub = "age1teupt3wxdyz454jmdf09c6387hafkg26tr8eqm9tawv53p29rfaqjq0dvu";
ageKey = "AGE-SECRET-KEY-1XYQSNDSZ8E8GJRK2W5ATE5U58M6VQMC0Y0NVVA3RKQ9FUXEKTP9QR4Q3Z5";
sops-nix = pkgs.fetchzip {
url = "https://github.com/Mic92/sops-nix/archive/${commit}.tar.gz";
hash = "sha256-t+voe2961vCgrzPFtZxha0/kmFSHFobzF00sT8p9h0U=";
};
secrets = pkgs.writeText "secrets.yaml" "password: aliceiscool2004";
secretsEnc = pkgs.runCommand "secrets-enc.yaml" { } ''
${lib.getExe pkgs.sops} encrypt ${secrets} --age "${agePub}" > $out
'';
in
{
name = "rclone-sops-nix";
nodes.machine = {
imports = [ ../../../../nixos ];
virtualisation.memorySize = 2048;
users.users.alice = {
isNormalUser = true;
description = "Alice Foobar";
password = "foobar";
uid = 1000;
};
systemd.tmpfiles.rules = [
"f /home/alice/age-key 400 alice users - ${ageKey}"
];
home-manager.users.alice =
{ config, ... }:
{
imports = [ "${sops-nix}/modules/home-manager/sops.nix" ];
home = {
username = "alice";
homeDirectory = "/home/alice";
stateVersion = "24.05"; # Please read the comment before changing.
};
sops = {
age.keyFile = "/home/alice/age-key";
secrets.password.sopsFile = secretsEnc;
};
programs.rclone = {
enable = true;
remotes = {
alices-encrypted-files = {
config = {
type = "crypt";
remote = "/home/alice/enc-files";
};
secrets.password = config.sops.secrets.password.path;
};
};
};
};
};
testScript = ''
def alice_cmd(cmd):
return f"su -l alice --shell /bin/sh -c $'export XDG_RUNTIME_DIR=/run/user/$UID ; {cmd}'"
def succeed_as_alice(*cmds, box=machine):
return box.succeed(*map(alice_cmd,cmds))
def assert_list(cmd, expected_list, actual):
assert all([x in actual for x in expected_list]), \
f"""Expected {cmd} to contain \
[{" and ".join([x for x in expected_list if x not in actual])}], but got {actual}"""
start_all()
machine.wait_for_unit("network.target")
machine.wait_for_unit("multi-user.target")
machine.wait_until_tty_matches("1", "login: ")
machine.send_chars("alice\n")
machine.wait_until_tty_matches("1", "Password: ")
machine.send_chars("foobar\n")
machine.wait_until_tty_matches("1", "alice\\@machine")
with subtest("Sops-nix activation ordering works correctly"):
# wait for rclone-config.service
machine.wait_until_succeeds("grep password /home/alice/.config/rclone/rclone.conf")
actual = succeed_as_alice("cat /home/alice/.config/rclone/rclone.conf")
assert_list("/home/alice/.config/rclone/rclone.conf", [
"[alices-encrypted-files]",
"remote = /home/alice/enc-files",
"type = crypt",
"password = "
], actual)
hidden_password = actual.strip().split("password = ")[1]
password = succeed_as_alice(f"rclone reveal {hidden_password}")
assert "aliceiscool2004" in password, \
f"Failed to decrypt password. Instead of aliceiscool2004, we got {password}"
'';
}

View file

@ -32,8 +32,8 @@ in
succeed_as_alice("install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix")
actual = succeed_as_alice("home-manager switch")
expected = "Activating createRcloneConfig"
assert expected in actual, \
expected = "rclone-config.service"
assert "Starting units: " in actual and expected in actual, \
f"expected home-manager switch to contain {expected}, but got {actual}"
succeed_as_alice("diff -u ${expected} /home/alice/.config/rclone/rclone.conf")

View file

@ -0,0 +1,23 @@
{ pkgs, ... }:
let
module = pkgs.writeText "write-after-module" ''
{
programs.rclone.writeAfter = "something";
}
'';
in
{
script = ''
with subtest("Use removed `writeAfter` option"):
succeed_as_alice("install -m644 ${module} /home/alice/.config/home-manager/test-remote.nix")
actual = fail_as_alice("home-manager switch 2>&1")
expected = "rclone-config.service"
assert expected not in actual, \
f"expected home-manager switch to not contain {expected}, but got {actual}"
snippet = "The writeAfter option has been removed because"
assert snippet in actual, \
f"expected home-manager switch to not contain {snippet}, but got {actual}"
'';
}