pasta: wip

TODO: add original authors as commit authors

src/libstore/unix/build/linux-derivation-builder.cc

@@ -1,9 +1,13 @@
 #ifdef __linux__
 
+#  include "nix/store/globals.hh"
 #  include "nix/store/personality.hh"
 #  include "nix/util/cgroup.hh"
+#  include "nix/util/file-system.hh"
 #  include "nix/util/linux-namespaces.hh"
+#  include "nix/util/strings.hh"
 #  include "linux/fchmodat2-compat.hh"
+#  include "pasta.hh"
 
 #  include <sys/ioctl.h>
 #  include <net/if.h>
@@ -190,6 +194,16 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
      */
     std::optional<Path> cgroup;
 
+    /**
+     * Process ID of pasta, if we're using it for network isolation.
+     */
+    Pid pastaPid;
+
+    /**
+     * Whether pasta was started for this build.
+     */
+    bool runPasta = false;
+
     ChrootLinuxDerivationBuilder(
         LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
         : DerivationBuilderImpl{store, std::move(miscMethods), std::move(params)}
@@ -198,6 +212,18 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
     {
     }
 
+    ~ChrootLinuxDerivationBuilder()
+    {
+        // pasta being left around mostly happens when builds are aborted
+        if (pastaPid) {
+            try {
+                pasta::killPasta(pastaPid);
+            } catch (Error & e) {
+                // Ignore errors during cleanup
+            }
+        }
+    }
+
     uid_t sandboxUid()
     {
         return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID();
@@ -433,6 +459,19 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1\n");
         userNamespaceSyncDone = true;
+
+        /* Set up pasta for network isolation if enabled and this is a fixed-output derivation */
+        bool privateNetwork = derivationType.isSandboxed();
+        runPasta = !privateNetwork && settings.pastaPath != "" && pathExists("/dev/net/tun");
+
+        if (runPasta) {
+            pastaPid = pasta::setupPasta(
+                settings.pastaPath,
+                pid,
+                buildUser ? std::optional(buildUser->getUID()) : std::nullopt,
+                buildUser ? std::optional(buildUser->getGID()) : std::nullopt,
+                usingUserNamespace);
+        }
     }
 
     void enterChroot() override
@@ -444,6 +483,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
 
         userNamespaceSync.readSide = -1;
 
+        if (runPasta) {
+            pasta::waitForPastaInterface();
+        }
+
         if (derivationType.isSandboxed()) {
 
             /* Initialise the loopback interface. */
@@ -532,8 +575,17 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
                happens when testing Nix building fixed-output derivations
                within a pure derivation. */
             for (auto & path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"})
-                if (pathExists(path))
-                    ss.push_back(path);
+                if (pathExists(path)) {
+                    // Special handling for resolv.conf when using pasta
+                    if (runPasta && std::string(path) == "/etc/resolv.conf") {
+                        // We'll write a modified version instead of bind-mounting
+                        auto resolvConf = readFile(std::string(path));
+                        auto modifiedResolvConf = pasta::rewriteResolvConf(resolvConf);
+                        writeFile(chrootRootDir + "/etc/resolv.conf", modifiedResolvConf);
+                    } else {
+                        ss.push_back(path);
+                    }
+                }
 
             if (settings.caFile != "") {
                 Path caFile = settings.caFile;
@@ -691,6 +743,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
             return;
         }
 
+        if (pastaPid) {
+            pasta::killPasta(pastaPid);
+        }
+
         DerivationBuilderImpl::killSandbox(getStats);
     }
 

src/libstore/unix/build/pasta.cc

@@ -0,0 +1,161 @@
+#ifdef __linux__
+
+#  include "pasta.hh"
+#  include "nix/util/error.hh"
+#  include "nix/util/file-system.hh"
+#  include "nix/util/processes.hh"
+#  include "nix/util/strings.hh"
+#  include "nix/util/util.hh"
+
+#  include "nix/util/fmt.hh"
+#  include <regex>
+#  include <sys/socket.h>
+#  include <net/if.h>
+#  include <sys/ioctl.h>
+#  include <unistd.h>
+#  include <fcntl.h>
+#  include <linux/capability.h>
+#  include <netinet/in.h>
+#  include <sys/types.h>
+
+namespace nix {
+namespace pasta {
+
+Pid setupPasta(
+    const Path & pastaPath,
+    pid_t pid,
+    std::optional<uid_t> buildUserId,
+    std::optional<gid_t> buildGroupId,
+    bool usingUserNamespace)
+{
+    // Bring up pasta for handling FOD networking. We don't let it daemonize
+    // itself for process management reasons and kill it manually when done.
+
+    Strings args = {
+        "--quiet",        "--foreground",    "--config-net",     "--gateway",       PASTA_HOST_IPV4, "--address",
+        PASTA_CHILD_IPV4, "--netmask",       PASTA_IPV4_NETMASK, "--dns-forward",   PASTA_HOST_IPV4, "--gateway",
+        PASTA_HOST_IPV6,  "--address",       PASTA_CHILD_IPV6,   "--dns-forward",   PASTA_HOST_IPV6, "--ns-ifname",
+        PASTA_NS_IFNAME,  "--no-netns-quit", "--netns",          "/proc/self/fd/0",
+    };
+
+    AutoCloseFD netns(open(fmt("/proc/%i/ns/net", pid).c_str(), O_RDONLY | O_CLOEXEC));
+    if (!netns) {
+        throw SysError("failed to open netns");
+    }
+
+    AutoCloseFD userns;
+    if (usingUserNamespace) {
+        userns = AutoCloseFD(open(fmt("/proc/%i/ns/user", pid).c_str(), O_RDONLY | O_CLOEXEC));
+        if (!userns) {
+            throw SysError("failed to open userns");
+        }
+        args.push_back("--userns");
+        args.push_back("/proc/self/fd/1");
+    }
+
+    // FIXME ideally we want a notification when pasta exits, but we cannot do
+    // this at present. Without such support we need to busy-wait for pasta to
+    // set up the namespace completely and time out after a while for the case
+    // of pasta launch failures. Pasta logs go to syslog only for now as well.
+
+    // Use startProcess to launch pasta with proper file descriptor setup
+    return startProcess([&]() {
+        // Set up file descriptor redirections
+        if (dup2(netns.get(), 0) == -1)
+            throw SysError("cannot redirect netns fd to stdin");
+
+        if (userns) {
+            if (dup2(userns.get(), 1) == -1)
+                throw SysError("cannot redirect userns fd to stdout");
+        }
+
+        // Set user/group if specified
+        if (buildGroupId && setgid(*buildGroupId) == -1)
+            throw SysError("setgid failed");
+        if (buildUserId && setuid(*buildUserId) == -1)
+            throw SysError("setuid failed");
+
+        // Execute pasta
+        Strings allArgs = {pastaPath};
+        allArgs.insert(allArgs.end(), args.begin(), args.end());
+
+        execvp(pastaPath.c_str(), stringsToCharPtrs(allArgs).data());
+        throw SysError("executing pasta");
+    });
+}
+
+void waitForPastaInterface()
+{
+    // Wait for the pasta interface to appear. pasta can't signal us when
+    // it's done setting up the namespace, so we have to wait for a while
+    AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+    if (!fd)
+        throw SysError("cannot open IP socket");
+
+    struct ifreq ifr;
+    strcpy(ifr.ifr_name, PASTA_NS_IFNAME);
+
+    // Wait two minutes for the interface to appear. If it does not do so
+    // we are either grossly overloaded, or pasta startup failed somehow.
+    static constexpr int SINGLE_WAIT_US = 1000;
+    static constexpr int TOTAL_WAIT_US = 120'000'000;
+
+    for (unsigned tries = 0;; tries++) {
+        if (tries > TOTAL_WAIT_US / SINGLE_WAIT_US) {
+            throw Error(
+                "sandbox network setup timed out, please check daemon logs for "
+                "possible error output.");
+        } else if (ioctl(fd.get(), SIOCGIFFLAGS, &ifr) == 0) {
+            if ((ifr.ifr_ifru.ifru_flags & IFF_UP) != 0) {
+                break;
+            }
+        } else if (errno == ENODEV) {
+            usleep(SINGLE_WAIT_US);
+        } else {
+            throw SysError("cannot get loopback interface flags");
+        }
+    }
+}
+
+std::string rewriteResolvConf(const std::string & fromHost)
+{
+    static constexpr auto flags = std::regex::ECMAScript | std::regex::multiline;
+    static std::regex lineRegex("^nameserver\\s.*$", flags);
+    static std::regex v4Regex("^nameserver\\s+\\d{1,3}\\.", flags);
+    static std::regex v6Regex("^nameserver.*:", flags);
+
+    std::string nsInSandbox = "\n";
+    if (std::regex_search(fromHost, v4Regex)) {
+        nsInSandbox += nix::fmt("nameserver %s\n", PASTA_HOST_IPV4);
+    }
+    if (std::regex_search(fromHost, v6Regex)) {
+        nsInSandbox += nix::fmt("nameserver %s\n", PASTA_HOST_IPV6);
+    }
+
+    return std::regex_replace(fromHost, lineRegex, "") + nsInSandbox;
+}
+
+void killPasta(Pid & pastaPid)
+{
+    // FIXME we really want to send SIGTERM instead and wait for pasta to exit,
+    // but we do not have the infra for that right now. We send SIGKILL instead
+    // and treat exiting with that as a successful exit code until such a time.
+    // This is not likely to cause problems since pasta runs as the build user,
+    // but not inside the build sandbox. If it's killed it's either due to some
+    // external influence (in which case the sandboxed child will probably fail
+    // due to network errors, if it used the network at all) or some bug in nix
+    if (auto status = pastaPid.kill(); !WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) {
+        if (WIFSIGNALED(status)) {
+            throw Error("pasta killed by signal %i", WTERMSIG(status));
+        } else if (WIFEXITED(status)) {
+            throw Error("pasta exited with code %i", WEXITSTATUS(status));
+        } else {
+            throw Error("pasta exited with status %i", status);
+        }
+    }
+}
+
+} // namespace pasta
+} // namespace nix
+
+#endif // __linux__

src/libstore/unix/build/pasta.hh

@@ -0,0 +1,78 @@
+#pragma once
+
+#ifdef __linux__
+
+#  include "nix/util/processes.hh"
+#  include "nix/util/file-descriptor.hh"
+#  include <string>
+#  include <set>
+
+namespace nix {
+
+/**
+ * Pasta (Plug A Simple Socket Transport) network isolation constants and utilities.
+ *
+ * Pasta provides network isolation for fixed-output derivations by creating
+ * a Layer-2 to Layer-4 translation without requiring special privileges.
+ */
+namespace pasta {
+
+// Network configuration constants
+// NOTE: These are all C strings because macOS doesn't have constexpr std::string
+// constructors, and std::string_view is a pain to turn into std::strings again.
+static constexpr const char * PASTA_NS_IFNAME = "eth0";
+static constexpr const char * PASTA_HOST_IPV4 = "169.254.1.1";
+static constexpr const char * PASTA_CHILD_IPV4 = "169.254.1.2";
+static constexpr const char * PASTA_IPV4_NETMASK = "16";
+// Randomly chosen 6to4 prefix, mapping the same ipv4ll as above.
+// Even if this id is used on the daemon host there should not be
+// any collisions since ipv4ll should never be addressed by ipv6.
+static constexpr const char * PASTA_HOST_IPV6 = "64:ff9b:1:4b8e:472e:a5c8:a9fe:0101";
+static constexpr const char * PASTA_CHILD_IPV6 = "64:ff9b:1:4b8e:472e:a5c8:a9fe:0102";
+
+/**
+ * Launch pasta for network isolation of a build process.
+ *
+ * @param pastaPath Path to the pasta executable
+ * @param pid Process ID of the build process to isolate
+ * @param buildUserId Optional UID to run pasta as
+ * @param buildGroupId Optional GID to run pasta as
+ * @param usingUserNamespace Whether the build is using a user namespace
+ * @return Process ID of the pasta process
+ */
+Pid setupPasta(
+    const Path & pastaPath,
+    pid_t pid,
+    std::optional<uid_t> buildUserId,
+    std::optional<gid_t> buildGroupId,
+    bool usingUserNamespace);
+
+/**
+ * Wait for pasta to set up the network interface.
+ *
+ * @throws Error if the interface doesn't appear within the timeout period
+ */
+void waitForPastaInterface();
+
+/**
+ * Rewrite /etc/resolv.conf for pasta-isolated builds.
+ *
+ * Replaces nameserver entries with pasta's DNS forwarders.
+ *
+ * @param fromHost The original resolv.conf content from the host
+ * @return Modified resolv.conf content for the sandboxed build
+ */
+std::string rewriteResolvConf(const std::string & fromHost);
+
+/**
+ * Kill the pasta process.
+ *
+ * @param pastaPid The pasta process to kill
+ * @throws Error if pasta exits with unexpected status
+ */
+void killPasta(Pid & pastaPid);
+
+} // namespace pasta
+} // namespace nix
+
+#endif // __linux__

src/libstore/unix/meson.build

@@ -6,4 +6,16 @@ sources += files(
   'user-lock.cc',
 )
 
+if host_machine.system() == 'linux'
+  sources += files(
+    'build/pasta.cc',
+  )
+endif
+
+if host_machine.system() == 'darwin'
+  sources += files(
+    'build/darwin-derivation-builder.cc',
+  )
+endif
+
 subdir('include/nix/store')