Add sandboxed building for FreeBSD using jails

Co-Authored-By: Artemis Tosini <me@artem.ist>
Co-Authored-By: John Ericson <John.Ericson@Obsidian.Systems>

src/libstore/meson.build

@@ -365,6 +365,11 @@ this_library = library(
 install_headers(headers, subdir : 'nix/store', preserve_path : true)
 
 libraries_private = []
+# `libraries_private` cannot contain ad-hoc dependencies (from
+# `find_library), so we need to do this manually
+if host_machine.system() == 'freebsd'
+  libraries_private += [ '-ljail' ]
+endif
 
 extra_pkg_config_variables = {
   'storedir' : get_option('store-dir'),

src/libstore/package.nix

@@ -5,6 +5,7 @@
 
   unixtools,
   darwin,
+  freebsd,
 
   nix-util,
   boost,
@@ -67,6 +68,7 @@ mkMesonLibrary (finalAttrs: {
     ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
     # There have been issues building these dependencies
     ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
+    ++ lib.optional stdenv.hostPlatform.isFreeBSD freebsd.libjail
     ++ lib.optional withAWS aws-sdk-cpp;
 
   propagatedBuildInputs = [

src/libstore/unix/build/chroot-derivation-builder.cc

@@ -1,4 +1,4 @@
-#ifdef __linux__
+#if defined(__linux__) || defined(__FreeBSD__)
 
 namespace nix {
 
@@ -59,6 +59,8 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
         return buildUser->getGID();
     }
 
+    virtual void extraChrootParentDirCleanup(const Path & chrootParentDir) {}
+
     void prepareSandbox() override
     {
         /* Create a temporary directory in which we set up the chroot
@@ -66,6 +68,7 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
            so that the build outputs can be moved efficiently from the
            chroot to their final location. */
         auto chrootParentDir = store.Store::toRealPath(drvPath) + ".chroot";
+        extraChrootParentDirCleanup(chrootParentDir);
         deletePath(chrootParentDir);
 
         /* Clean up the chroot directory automatically. */

src/libstore/unix/build/derivation-builder.cc

@@ -2159,6 +2159,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
 // FIXME: do this properly
 #include "chroot-derivation-builder.cc"
 #include "linux-derivation-builder.cc"
+#include "freebsd-derivation-builder.cc"
 #include "darwin-derivation-builder.cc"
 
 namespace nix {
@@ -2221,6 +2222,11 @@ std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
         return std::make_unique<ChrootLinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
 
     return std::make_unique<LinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
+#elif defined(__FreeBSD__)
+    if (useSandbox)
+        return std::make_unique<ChrootFreeBSDDerivationBuilder>(store, std::move(miscMethods), std::move(params));
+
+    return std::make_unique<FreeBSDDerivationBuilder>(store, std::move(miscMethods), std::move(params));
 #else
     if (useSandbox)
         throw Error("sandboxing builds is not supported on this platform");

src/libstore/unix/build/freebsd-derivation-builder.cc

@@ -0,0 +1,220 @@
+#ifdef __FreeBSD__
+
+#  include <sys/param.h>
+#  include <sys/jail.h>
+#  include <jail.h>
+#  include <stdlib.h>
+#  include <sys/mount.h>
+#  include <string.h>
+#  include "nix/util/freebsd-jail.hh"
+
+namespace nix {
+
+struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl
+{
+    using DerivationBuilderImpl::DerivationBuilderImpl;
+};
+
+struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder
+{
+    /* Destructors happen in reverse order from declaration */
+    std::shared_ptr<AutoRemoveJail> autoDelJail;
+    std::vector<std::shared_ptr<AutoUnmount>> autoDelMounts;
+
+    ChrootFreeBSDDerivationBuilder(
+        Store & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
+        : DerivationBuilderImpl{store, std::move(miscMethods), std::move(params)}
+        , ChrootDerivationBuilder{store, std::move(miscMethods), std::move(params)}
+        , FreeBSDDerivationBuilder{store, std::move(miscMethods), std::move(params)}
+    {
+    }
+
+    void deleteTmpDir(bool force) override
+    {
+        /* Unmount and free jail id, if in use */
+        autoDelMounts.clear();
+        autoDelJail.reset();
+
+        ChrootDerivationBuilder::deleteTmpDir(force);
+    }
+
+    void extraChrootParentDirCleanup(const Path & chrootParentDir) override
+    {
+        int count;
+        struct statfs * mntbuf;
+        if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
+            throw SysError("Couldn't get mount info for chroot");
+        }
+
+        for (int i = 0; i < count; i++) {
+            Path mounted(mntbuf[i].f_mntonname);
+            if (hasPrefix(mounted, chrootParentDir)) {
+                if (unmount(mounted.c_str(), 0) < 0) {
+                    throw SysError("Failed to unmount path %1%", mounted);
+                }
+            }
+        }
+    }
+
+    void prepareSandbox() override
+    {
+        auto devpath = chrootRootDir + "/dev";
+        mkdir(devpath.c_str(), 0555);
+        mkdir((chrootRootDir + "/bin").c_str(), 0555);
+        char errmsg[255] = "";
+        struct iovec iov[8] = {
+            {.iov_base = (void *) "fstype", .iov_len = sizeof("fstype")},
+            {.iov_base = (void *) "devfs", .iov_len = sizeof("devfs")},
+            {.iov_base = (void *) "fspath", .iov_len = sizeof("fspath")},
+            {.iov_base = (void *) devpath.c_str(), .iov_len = devpath.length() + 1},
+            {.iov_base = (void *) "errmsg", .iov_len = sizeof("errmsg")},
+            {.iov_base = (void *) errmsg, .iov_len = sizeof(errmsg)},
+        };
+        if (nmount(iov, 6, 0) < 0) {
+            throw SysError("Failed to mount jail /dev: %1%", errmsg);
+        }
+        autoDelMounts.push_back(std::make_shared<AutoUnmount>(devpath));
+
+        /* Fixed-output derivations typically need to access the
+           network, so give them access to /etc/resolv.conf and so
+           on. */
+        if (!derivationType.isSandboxed()) {
+            // Only use nss functions to resolve hosts and
+            // services. Don’t use it for anything else that may
+            // be configured for this system. This limits the
+            // potential impurities introduced in fixed-outputs.
+            writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
+
+            /* N.B. it is realistic that these paths might not exist. It
+               happens when testing Nix building fixed-output derivations
+               within a pure derivation. */
+            for (auto & path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"})
+                if (pathExists(path))
+                    pathsInChroot.try_emplace(path, path, true);
+
+            if (settings.caFile != "")
+                pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
+        }
+
+        for (auto & i : pathsInChroot) {
+            char errmsg[255];
+            errmsg[0] = 0;
+
+            if (i.second.source == "/proc")
+                continue; // backwards compatibility
+            auto path = chrootRootDir + i.first;
+
+            struct stat stat_buf;
+            if (stat(i.second.source.c_str(), &stat_buf) < 0) {
+                throw SysError("stat");
+            }
+
+            // mount points must exist and be the right type
+            if (S_ISDIR(stat_buf.st_mode)) {
+                mkdir(path.c_str(), 0555);
+            } else {
+                close(open(path.c_str(), O_CREAT | O_RDONLY, 0444));
+            }
+
+            struct iovec iov[8] = {
+                {.iov_base = (void *) "fstype", .iov_len = sizeof("fstype")},
+                {.iov_base = (void *) "nullfs", .iov_len = sizeof("nullfs")},
+                {.iov_base = (void *) "fspath", .iov_len = sizeof("fspath")},
+                {.iov_base = (void *) path.c_str(), .iov_len = path.length() + 1},
+                {.iov_base = (void *) "target", .iov_len = sizeof("target")},
+                {.iov_base = (void *) i.second.source.c_str(), .iov_len = i.second.source.length() + 1},
+                {.iov_base = (void *) "errmsg", .iov_len = sizeof("errmsg")},
+                {.iov_base = (void *) errmsg, .iov_len = sizeof(errmsg)},
+            };
+            if (nmount(iov, 8, 0) < 0) {
+                throw SysError("Failed to mount nullfs for %1% - %2%", path, errmsg);
+            }
+            autoDelMounts.push_back(std::make_shared<AutoUnmount>(path));
+        }
+    }
+
+    void startChild() override
+    {
+        /* Now that we now the sandbox uid, we can write
+           /etc/passwd. */
+        writeFile(
+            chrootRootDir + "/etc/passwd",
+            fmt("root:x:0:0::::Nix build user:%3%:/noshell\n"
+                "nixbld:x:%1%:%2%::::Nix build user:%3%:/noshell\n"
+                "nobody:x:65534:65534::::Nobody:/:/noshell\n",
+                buildUser->getUID(),
+                sandboxGid(),
+                settings.sandboxBuildDir));
+        if (system(("pwd_mkdb -d " + chrootRootDir + "/etc " + chrootRootDir + "/etc/passwd 2>/dev/null").c_str())
+            != 0) {
+            throw SysError("Failed to set up isolated users");
+        }
+
+        int jid;
+
+        if (derivationType.isSandboxed()) {
+            jid = jail_setv(
+                JAIL_CREATE,
+                "persist",
+                "true",
+                "path",
+                chrootRootDir.c_str(),
+                "devfs_ruleset",
+                "4",
+                "vnet",
+                "new",
+                "host.hostname",
+                "nixbsd",
+                NULL);
+            if (jid < 0) {
+                throw SysError("Failed to create jail (isolated network)");
+            }
+            autoDelJail = std::make_shared<AutoRemoveJail>(jid);
+
+            if (system(("ifconfig -j " + std::to_string(jid) + " lo0 inet 127.0.0.1/8 up").c_str()) != 0) {
+                throw SysError("Failed to set up isolated network");
+            }
+        } else {
+            jid = jail_setv(
+                JAIL_CREATE,
+                "persist",
+                "true",
+                "path",
+                chrootRootDir.c_str(),
+                "devfs_ruleset",
+                "4",
+                "ip4",
+                "inherit",
+                "ip6",
+                "inherit",
+                "allow.raw_sockets",
+                "true",
+                "host.hostname",
+                "nixbsd",
+                NULL);
+            if (jid < 0) {
+                throw SysError("Failed to create jail (fixed-derivation)");
+            }
+            autoDelJail = std::make_shared<AutoRemoveJail>(jid);
+        }
+
+        pid = startProcess([&]() {
+            openSlave();
+            if (jail_attach(jid) < 0) {
+                throw SysError("Failed to attach to jail");
+            }
+            runChild();
+        });
+    }
+
+    void addDependency(const StorePath & path) override
+    {
+        auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path);
+
+        warn("Not yet implemented, dependency not added inside sandbox");
+    }
+};
+
+} // namespace nix
+
+#endif