Merge remote-tracking branch 'origin/2.29-maintenance' into detsys-main

src/libstore/unix/include/nix/store/build/derivation-builder.hh

@@ -0,0 +1,209 @@
+#pragma once
+///@file
+
+#include "nix/store/build-result.hh"
+#include "nix/store/derivation-options.hh"
+#include "nix/store/build/derivation-building-misc.hh"
+#include "nix/store/derivations.hh"
+#include "nix/store/parsed-derivations.hh"
+#include "nix/util/processes.hh"
+#include "nix/store/restricted-store.hh"
+#include "nix/store/user-lock.hh"
+
+namespace nix {
+
+/**
+ * Parameters by (mostly) `const` reference for `DerivationBuilder`.
+ */
+struct DerivationBuilderParams
+{
+    /** The path of the derivation. */
+    const StorePath & drvPath;
+
+    BuildResult & buildResult;
+
+    /**
+     * The derivation stored at drvPath.
+     */
+    const Derivation & drv;
+
+    /**
+     * The "structured attrs" of `drv`, if it has them.
+     *
+     * @todo this should be part of `Derivation`.
+     *
+     * @todo this should be renamed from `parsedDrv`.
+     */
+    const StructuredAttrs * parsedDrv;
+
+    /**
+     * The derivation options of `drv`.
+     *
+     * @todo this should be part of `Derivation`.
+     */
+    const DerivationOptions & drvOptions;
+
+    // The remainder is state held during the build.
+
+    /**
+     * All input paths (that is, the union of FS closures of the
+     * immediate input paths).
+     */
+    const StorePathSet & inputPaths;
+
+    /**
+     * @note we do in fact mutate this
+     */
+    std::map<std::string, InitialOutput> & initialOutputs;
+
+    const BuildMode & buildMode;
+
+    DerivationBuilderParams(
+        const StorePath & drvPath,
+        const BuildMode & buildMode,
+        BuildResult & buildResult,
+        const Derivation & drv,
+        const StructuredAttrs * parsedDrv,
+        const DerivationOptions & drvOptions,
+        const StorePathSet & inputPaths,
+        std::map<std::string, InitialOutput> & initialOutputs)
+        : drvPath{drvPath}
+        , buildResult{buildResult}
+        , drv{drv}
+        , parsedDrv{parsedDrv}
+        , drvOptions{drvOptions}
+        , inputPaths{inputPaths}
+        , initialOutputs{initialOutputs}
+        , buildMode{buildMode}
+    { }
+
+    DerivationBuilderParams(DerivationBuilderParams &&) = default;
+};
+
+/**
+ * Callbacks that `DerivationBuilder` needs.
+ */
+struct DerivationBuilderCallbacks
+{
+    virtual ~DerivationBuilderCallbacks() = default;
+
+    /**
+     * Open a log file and a pipe to it.
+     */
+    virtual Path openLogFile() = 0;
+
+    /**
+     * Close the log file.
+     */
+    virtual void closeLogFile() = 0;
+
+    /**
+     * Aborts if any output is not valid or corrupt, and otherwise
+     * returns a 'SingleDrvOutputs' structure containing all outputs.
+     *
+     * @todo Probably should just be in `DerivationGoal`.
+     */
+    virtual SingleDrvOutputs assertPathValidity() = 0;
+
+    virtual void appendLogTailErrorMsg(std::string & msg) = 0;
+
+    /**
+     * Hook up `builderOut` to some mechanism to ingest the log
+     *
+     * @todo this should be reworked
+     */
+    virtual void childStarted(Descriptor builderOut) = 0;
+
+    /**
+     * @todo this should be reworked
+     */
+    virtual void childTerminated() = 0;
+
+    virtual void noteHashMismatch(void) = 0;
+    virtual void noteCheckMismatch(void) = 0;
+
+    virtual void markContentsGood(const StorePath & path) = 0;
+};
+
+/**
+ * This class represents the state for building locally.
+ *
+ * @todo Ideally, it would not be a class, but a single function.
+ * However, besides the main entry point, there are a few more methods
+ * which are externally called, and need to be gotten rid of. There are
+ * also some virtual methods (either directly here or inherited from
+ * `DerivationBuilderCallbacks`, a stop-gap) that represent outgoing
+ * rather than incoming call edges that either should be removed, or
+ * become (higher order) function parameters.
+ */
+struct DerivationBuilder : RestrictionContext
+{
+    /**
+     * User selected for running the builder.
+     */
+    std::unique_ptr<UserLock> buildUser;
+
+    /**
+     * The process ID of the builder.
+     */
+    Pid pid;
+
+    DerivationBuilder() = default;
+    virtual ~DerivationBuilder() = default;
+
+    /**
+     * Master side of the pseudoterminal used for the builder's
+     * standard output/error.
+     */
+    AutoCloseFD builderOut;
+
+    /**
+     * Set up build environment / sandbox, acquiring resources (e.g.
+     * locks as needed). After this is run, the builder should be
+     * started.
+     *
+     * @returns true if successful, false if we could not acquire a build
+     * user. In that case, the caller must wait and then try again.
+     */
+    virtual bool prepareBuild() = 0;
+
+    /**
+     * Start building a derivation.
+     */
+    virtual void startBuilder() = 0;
+
+    /**
+     * Tear down build environment after the builder exits (either on
+     * its own or if it is killed).
+     *
+     * @returns The first case indicates failure during output
+     * processing. A status code and exception are returned, providing
+     * more information. The second case indicates success, and
+     * realisations for each output of the derivation are returned.
+     */
+    virtual std::variant<std::pair<BuildResult::Status, Error>, SingleDrvOutputs> unprepareBuild() = 0;
+
+    /**
+     * Stop the in-process nix daemon thread.
+     * @see startDaemon
+     */
+    virtual void stopDaemon() = 0;
+
+    /**
+     * Delete the temporary directory, if we have one.
+     */
+    virtual void deleteTmpDir(bool force) = 0;
+
+    /**
+     * Kill any processes running under the build user UID or in the
+     * cgroup of the build.
+     */
+    virtual void killSandbox(bool getStats) = 0;
+};
+
+std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
+    Store & store,
+    std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
+    DerivationBuilderParams params);
+
+}

src/libstore/unix/include/nix/store/build/local-derivation-goal.hh

@@ -1,319 +0,0 @@
-#pragma once
-///@file
-
-#include "nix/store/build/derivation-goal.hh"
-#include "nix/store/local-store.hh"
-#include "nix/util/processes.hh"
-
-namespace nix {
-
-struct LocalDerivationGoal : public DerivationGoal
-{
-    LocalStore & getLocalStore();
-
-    /**
-     * User selected for running the builder.
-     */
-    std::unique_ptr<UserLock> buildUser;
-
-    /**
-     * The process ID of the builder.
-     */
-    Pid pid;
-
-    /**
-     * The cgroup of the builder, if any.
-     */
-    std::optional<Path> cgroup;
-
-    /**
-     * The temporary directory used for the build.
-     */
-    Path tmpDir;
-
-    /**
-     * The top-level temporary directory. `tmpDir` is either equal to
-     * or a child of this directory.
-     */
-    Path topTmpDir;
-
-    /**
-     * The path of the temporary directory in the sandbox.
-     */
-    Path tmpDirInSandbox;
-
-    /**
-     * Master side of the pseudoterminal used for the builder's
-     * standard output/error.
-     */
-    AutoCloseFD builderOut;
-
-    /**
-     * Pipe for synchronising updates to the builder namespaces.
-     */
-    Pipe userNamespaceSync;
-
-    /**
-     * The mount namespace and user namespace of the builder, used to add additional
-     * paths to the sandbox as a result of recursive Nix calls.
-     */
-    AutoCloseFD sandboxMountNamespace;
-    AutoCloseFD sandboxUserNamespace;
-
-    /**
-     * On Linux, whether we're doing the build in its own user
-     * namespace.
-     */
-    bool usingUserNamespace = true;
-
-    /**
-     * Whether we're currently doing a chroot build.
-     */
-    bool useChroot = false;
-
-    /**
-     * The parent directory of `chrootRootDir`. It has permission 700
-     * and is owned by root to ensure other users cannot mess with
-     * `chrootRootDir`.
-     */
-    Path chrootParentDir;
-
-    /**
-     * The root of the chroot environment.
-     */
-    Path chrootRootDir;
-
-    /**
-     * RAII object to delete the chroot directory.
-     */
-    std::shared_ptr<AutoDelete> autoDelChroot;
-
-    /**
-     * Whether to run the build in a private network namespace.
-     */
-    bool privateNetwork = false;
-
-    /**
-     * Stuff we need to pass to initChild().
-     */
-    struct ChrootPath {
-        Path source;
-        bool optional;
-        ChrootPath(Path source = "", bool optional = false)
-            : source(source), optional(optional)
-        { }
-    };
-    typedef map<Path, ChrootPath> PathsInChroot; // maps target path to source path
-    PathsInChroot pathsInChroot;
-
-    typedef map<std::string, std::string> Environment;
-    Environment env;
-
-    /**
-     * Hash rewriting.
-     */
-    StringMap inputRewrites, outputRewrites;
-    typedef map<StorePath, StorePath> RedirectedOutputs;
-    RedirectedOutputs redirectedOutputs;
-
-    /**
-     * The output paths used during the build.
-     *
-     * - Input-addressed derivations or fixed content-addressed outputs are
-     *   sometimes built when some of their outputs already exist, and can not
-     *   be hidden via sandboxing. We use temporary locations instead and
-     *   rewrite after the build. Otherwise the regular predetermined paths are
-     *   put here.
-     *
-     * - Floating content-addressing derivations do not know their final build
-     *   output paths until the outputs are hashed, so random locations are
-     *   used, and then renamed. The randomness helps guard against hidden
-     *   self-references.
-     */
-    OutputPathMap scratchOutputs;
-
-    uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100  : 0) : buildUser->getGID(); }
-
-    const static Path homeDir;
-
-    /**
-     * The recursive Nix daemon socket.
-     */
-    AutoCloseFD daemonSocket;
-
-    /**
-     * The daemon main thread.
-     */
-    std::thread daemonThread;
-
-    /**
-     * The daemon worker threads.
-     */
-    std::vector<std::thread> daemonWorkerThreads;
-
-    /**
-     * Paths that were added via recursive Nix calls.
-     */
-    StorePathSet addedPaths;
-
-    /**
-     * Realisations that were added via recursive Nix calls.
-     */
-    std::set<DrvOutput> addedDrvOutputs;
-
-    /**
-     * Recursive Nix calls are only allowed to build or realize paths
-     * in the original input closure or added via a recursive Nix call
-     * (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
-     * /nix/store/<bla> is some arbitrary path in a binary cache).
-     */
-    bool isAllowed(const StorePath & path)
-    {
-        return inputPaths.count(path) || addedPaths.count(path);
-    }
-    bool isAllowed(const DrvOutput & id)
-    {
-        return addedDrvOutputs.count(id);
-    }
-
-    bool isAllowed(const DerivedPath & req);
-
-    friend struct RestrictedStore;
-
-    using DerivationGoal::DerivationGoal;
-
-    virtual ~LocalDerivationGoal() override;
-
-    /**
-     * Whether we need to perform hash rewriting if there are valid output paths.
-     */
-    bool needsHashRewrite();
-
-    /**
-     * The additional states.
-     */
-    Goal::Co tryLocalBuild() override;
-
-    /**
-     * Start building a derivation.
-     */
-    void startBuilder();
-
-    /**
-     * Fill in the environment for the builder.
-     */
-    void initEnv();
-
-    /**
-     * Process messages send by the sandbox initialization.
-     */
-    void processSandboxSetupMessages();
-
-    /**
-     * Setup tmp dir location.
-     */
-    void initTmpDir();
-
-    /**
-     * Write a JSON file containing the derivation attributes.
-     */
-    void writeStructuredAttrs();
-
-    /**
-     * Start an in-process nix daemon thread for recursive-nix.
-     */
-    void startDaemon();
-
-    /**
-     * Stop the in-process nix daemon thread.
-     * @see startDaemon
-     */
-    void stopDaemon();
-
-    /**
-     * Add 'path' to the set of paths that may be referenced by the
-     * outputs, and make it appear in the sandbox.
-     */
-    void addDependency(const StorePath & path);
-
-    /**
-     * Make a file owned by the builder.
-     */
-    void chownToBuilder(const Path & path);
-
-    int getChildStatus() override;
-
-    /**
-     * Run the builder's process.
-     */
-    void runChild();
-
-    /**
-     * Check that the derivation outputs all exist and register them
-     * as valid.
-     */
-    SingleDrvOutputs registerOutputs() override;
-
-    void signRealisation(Realisation &) override;
-
-    /**
-     * Check that an output meets the requirements specified by the
-     * 'outputChecks' attribute (or the legacy
-     * '{allowed,disallowed}{References,Requisites}' attributes).
-     */
-    void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
-
-    /**
-     * Close the read side of the logger pipe.
-     */
-    void closeReadPipes() override;
-
-    /**
-     * Cleanup hooks for buildDone()
-     */
-    void cleanupHookFinally() override;
-    void cleanupPreChildKill() override;
-    void cleanupPostChildKill() override;
-    bool cleanupDecideWhetherDiskFull() override;
-    void cleanupPostOutputsRegisteredModeCheck() override;
-    void cleanupPostOutputsRegisteredModeNonCheck() override;
-
-    bool isReadDesc(int fd) override;
-
-    /**
-     * Delete the temporary directory, if we have one.
-     */
-    void deleteTmpDir(bool force);
-
-    /**
-     * Forcibly kill the child process, if any.
-     *
-     * Called by destructor, can't be overridden
-     */
-    void killChild() override final;
-
-    /**
-     * Kill any processes running under the build user UID or in the
-     * cgroup of the build.
-     */
-    void killSandbox(bool getStats);
-
-    /**
-     * Create alternative path calculated from but distinct from the
-     * input, so we can avoid overwriting outputs (or other store paths)
-     * that already exist.
-     */
-    StorePath makeFallbackPath(const StorePath & path);
-
-    /**
-     * Make a path to another based on the output name along with the
-     * derivation hash.
-     *
-     * @todo Add option to randomize, so we can audit whether our
-     * rewrites caught everything
-     */
-    StorePath makeFallbackPath(OutputNameView outputName);
-};
-
-}

src/libstore/unix/include/nix/store/meson.build

@@ -2,7 +2,7 @@ include_dirs += include_directories('../..')
 
 headers += files(
   'build/child.hh',
+  'build/derivation-builder.hh',
   'build/hook-instance.hh',
-  'build/local-derivation-goal.hh',
   'user-lock.hh',
 )

src/libstore/unix/meson.build

@@ -1,7 +1,7 @@
 sources += files(
   'build/child.cc',
+  'build/derivation-builder.cc',
   'build/hook-instance.cc',
-  'build/local-derivation-goal.cc',
   'pathlocks.cc',
   'user-lock.cc',
 )