mirror of
https://github.com/NixOS/nix.git
synced 2025-11-09 12:06:01 +01:00
Add proof of concept daemon forwarding VM test
I was hoping for a more obviously correct solution in terms of security, but this is still a nice addition to the socket-only-daemon* functional tests. This could be used as a starting point for building out two things - Another method for running the functional tests, where the local Nix client is relocated and dependent on its remote builder. - An alternative, simpler solution to the SSH-based "darwin" linux-builder solution. It would still need a means for entering a shell for troubleshooting tasks, but presumably this could also be managed through a unix socket or something.
This commit is contained in:
Robert Hensing
parent
6a017a2a87
commit
624e0d247c
2 changed files with 219 additions and 0 deletions
|
|
@ -115,6 +115,8 @@ in
|
|||
|
||||
remoteBuildsSshNg = runNixOSTest ./remote-builds-ssh-ng.nix;
|
||||
|
||||
remoteBuildsPlainDaemon = runNixOSTest ./remote-builds-plain-daemon.nix;
|
||||
|
||||
}
|
||||
// lib.concatMapAttrs (
|
||||
nixVersion:
|
||||
|
|
|
|||
217
tests/nixos/remote-builds-plain-daemon.nix
Normal file
217
tests/nixos/remote-builds-plain-daemon.nix
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
# Test Nix's remote build feature with host-to-VM socket forwarding.
|
||||
# This tests that the host (test driver) can perform remote builds in a VM
|
||||
# using a socket connection, demonstrating the socket-only daemon functionality.
|
||||
|
||||
{
|
||||
config,
|
||||
hostPkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
# TCP port for the Nix daemon inside the VM
|
||||
daemonPort = 3049;
|
||||
|
||||
# The configuration of the VM builder.
|
||||
builder =
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{
|
||||
environment.systemPackages = [ pkgs.netcat ];
|
||||
virtualisation.writableStore = true;
|
||||
nix.settings.sandbox = true;
|
||||
|
||||
# Forward TCP port from host to guest
|
||||
# We'll use socat on the host to bridge Unix socket → localhost TCP
|
||||
# QEMU forwards localhost:daemonPort → guest:10.0.2.15:daemonPort
|
||||
# Note: QEMU will support Unix socket forwarding natively (hostfwd=unix:...)
|
||||
# once https://gitlab.com/qemu-project/qemu/-/commit/6d10e021318b16e3e90f98b7b2fa187826e26c0a
|
||||
# is released, which would eliminate the need for socat
|
||||
virtualisation.forwardPorts = [
|
||||
{
|
||||
from = "host";
|
||||
host.port = daemonPort;
|
||||
guest.port = daemonPort;
|
||||
}
|
||||
];
|
||||
|
||||
# Configure nix-daemon to listen on TCP directly
|
||||
# Empty string clears the default Unix socket, then we add TCP
|
||||
# Note: We bind to the eth0 IP address. In QEMU user networking, this is typically
|
||||
# 10.0.2.15 but we use FreeBind to allow binding before the network is fully configured.
|
||||
# Binding to a specific IP (not 0.0.0.0) prevents listening on localhost.
|
||||
# NOTE: This is similar to what is proposed in https://github.com/systemd/systemd/issues/32795
|
||||
# (using a separate network interface only accessible to systemd), but that's not yet
|
||||
# implemented.
|
||||
systemd.sockets.nix-daemon = {
|
||||
listenStreams = [
|
||||
""
|
||||
# QEMU user networking assigns 10.0.2.15 by default
|
||||
"10.0.2.15:${toString daemonPort}"
|
||||
];
|
||||
# FreeBind allows binding to IPs that don't exist yet
|
||||
socketConfig = {
|
||||
FreeBind = true;
|
||||
};
|
||||
};
|
||||
|
||||
# Restrict access to the daemon port: only allow connections from QEMU gateway
|
||||
# In QEMU user networking, forwarded connections appear to come from the gateway (10.0.2.2)
|
||||
# This should prevent unprivileged guest processes from accessing the daemon.
|
||||
# For production use, consider additional isolation mechanisms (see systemd.sockets comment above).
|
||||
# This has not been audited.
|
||||
networking.firewall.extraCommands = ''
|
||||
# Insert in reverse order since -I inserts at position 1
|
||||
# Drop all connections to daemon port (inserted first, will be at position 2)
|
||||
iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -j nixos-fw-log-refuse
|
||||
# Allow connections from QEMU gateway only (inserted second, will be at position 1)
|
||||
iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -s 10.0.2.2 -j nixos-fw-accept
|
||||
'';
|
||||
};
|
||||
|
||||
in
|
||||
|
||||
{
|
||||
config = {
|
||||
name = "remote-builds-plain-daemon";
|
||||
|
||||
nodes = {
|
||||
builder = builder;
|
||||
};
|
||||
|
||||
testScript =
|
||||
{ nodes }:
|
||||
''
|
||||
# fmt: off
|
||||
import subprocess
|
||||
import os
|
||||
import time
|
||||
|
||||
start_all()
|
||||
|
||||
# Wait for the VM to be ready
|
||||
builder.wait_for_unit("nix-daemon.socket")
|
||||
|
||||
# Verify the daemon is listening on TCP
|
||||
builder.succeed("ss -tlnp | grep ${toString daemonPort}")
|
||||
|
||||
print("VM builder is ready with TCP daemon")
|
||||
|
||||
# Start socat to bridge Unix socket → localhost TCP
|
||||
# QEMU forwards localhost:daemonPort → VM's 10.0.2.15:daemonPort
|
||||
# (See virtualisation.forwardPorts comment for future QEMU native Unix socket support)
|
||||
socket_path = os.environ.get('TMPDIR', '/tmp') + '/nix-builder.sock'
|
||||
|
||||
print(f"Starting socat to forward {socket_path} -> localhost:${toString daemonPort}")
|
||||
socat_proc = subprocess.Popen(
|
||||
["${hostPkgs.socat}/bin/socat",
|
||||
f"UNIX-LISTEN:{socket_path},fork",
|
||||
"TCP:127.0.0.1:${toString daemonPort}"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
# Wait for socket to be created
|
||||
for i in range(30):
|
||||
if os.path.exists(socket_path):
|
||||
break
|
||||
if socat_proc.poll() is not None:
|
||||
stdout, stderr = socat_proc.communicate()
|
||||
raise Exception(f"socat died unexpectedly: {stderr.decode()}")
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
socat_proc.terminate()
|
||||
raise Exception(f"Socket {socket_path} was not created by socat")
|
||||
|
||||
print(f"Host socket {socket_path} ready (socat -> QEMU -> VM)")
|
||||
|
||||
# Test the connection by trying a simple operation
|
||||
print("Testing connection to VM daemon...")
|
||||
subprocess.run(
|
||||
["${hostPkgs.nix}/bin/nix",
|
||||
"--extra-experimental-features", "nix-command",
|
||||
"--store", f"unix://{socket_path}",
|
||||
"store", "ping"],
|
||||
check=True,
|
||||
timeout=60
|
||||
)
|
||||
|
||||
# Create a simple derivation to build
|
||||
# Use a fixed system instead of builtins.currentSystem
|
||||
test_expr = """
|
||||
derivation {
|
||||
name = "socket-forward-host-build-test";
|
||||
system = "x86_64-linux";
|
||||
builder = "/bin/sh";
|
||||
args = [ "-c" "echo 'Built via forwarded socket from host!' > $out" ];
|
||||
}
|
||||
"""
|
||||
|
||||
expr_file = os.environ.get('TMPDIR', '/tmp') + '/test-expr.nix'
|
||||
with open(expr_file, 'w') as f:
|
||||
f.write(test_expr)
|
||||
|
||||
# Create a fresh store for the host
|
||||
host_store = os.environ.get('TMPDIR', '/tmp') + '/host-store'
|
||||
os.makedirs(host_store, exist_ok=True)
|
||||
|
||||
# Perform a build from the host using the VM as a builder
|
||||
# Builders format: <uri> <system> <ssh-key> <max-jobs> <speed-factor> <features> <mandatory-features>
|
||||
print("Host performing remote build in VM via socket->TCP bridge...")
|
||||
result = subprocess.run(
|
||||
["${hostPkgs.nix}/bin/nix-build",
|
||||
"--store", host_store,
|
||||
expr_file,
|
||||
"--no-out-link",
|
||||
"--option", "builders", f"unix://{socket_path} x86_64-linux - 1",
|
||||
"--option", "require-sigs", "false",
|
||||
"--max-jobs", "0"], # Force remote building
|
||||
stdout=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"Build failed with exit code {result.returncode}")
|
||||
raise Exception("Build failed")
|
||||
|
||||
out_path = result.stdout.strip()
|
||||
print(f"Build succeeded! Output: {out_path}")
|
||||
|
||||
# Verify the build happened in the VM by checking it exists there
|
||||
builder.succeed(f"test -e {out_path}")
|
||||
|
||||
# Verify the build output was copied to the host's physical store
|
||||
host_physical_path = f"{host_store}{out_path}"
|
||||
print(f"Checking host physical store at: {host_physical_path}")
|
||||
if not os.path.exists(host_physical_path):
|
||||
raise Exception(f"Build output not found in host store: {host_physical_path}")
|
||||
|
||||
# Verify the build output content
|
||||
with open(host_physical_path, 'r') as f:
|
||||
content = f.read()
|
||||
expected = "Built via forwarded socket from host!\n"
|
||||
if content != expected:
|
||||
raise Exception(f"Build output has wrong content. Expected: {repr(expected)}, got: {repr(content)}")
|
||||
|
||||
print("Socket-forwarded remote build from host test PASSED")
|
||||
|
||||
|
||||
# Test that guest processes CANNOT connect (firewall enabled)
|
||||
print("Testing that guest user CANNOT connect to daemon port (firewall enabled)...")
|
||||
builder.fail("timeout 5 nc -z 127.0.0.1 ${toString daemonPort}")
|
||||
print("Confirmed: guest user cannot connect to localhost")
|
||||
|
||||
builder.fail("timeout 5 nc -z 10.0.2.15 ${toString daemonPort}")
|
||||
print("Confirmed: guest user cannot connect to interface IP")
|
||||
|
||||
# Clean up socat
|
||||
print("Cleaning up socat...")
|
||||
socat_proc.terminate()
|
||||
socat_proc.wait(timeout=5)
|
||||
'';
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue