Merge remote-tracking branch 'origin/2.26-maintenance' into detsys-main

src/libstore/filetransfer.cc

@@ -300,6 +300,14 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
+        #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+        static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose) {
+            unix::closeOnExec(curlfd);
+            vomit("cloexec set for fd %i", curlfd);
+            return CURL_SOCKOPT_OK;
+        }
+        #endif
+
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -359,6 +367,10 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
+            #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+            curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
+            #endif
+
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);

src/libstore/globals.cc

@@ -65,7 +65,6 @@ Settings::Settings()
     , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
     , nixUserConfFiles(getUserConfigFiles())
-    , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
 #ifndef _WIN32
@@ -243,7 +242,7 @@ Path Settings::getDefaultSSLCertFile()
     return "";
 }
 
-const std::string nixVersion = PACKAGE_VERSION;
+std::string nixVersion = PACKAGE_VERSION;
 
 const std::string determinateNixVersion = DETERMINATE_NIX_VERSION;
 

src/libstore/globals.hh

@@ -84,11 +84,6 @@ public:
      */
     std::vector<Path> nixUserConfFiles;
 
-    /**
-     * The directory where the man pages are stored.
-     */
-    Path nixManDir;
-
     /**
      * File name of the socket the daemon listens to.
      */
@@ -1253,7 +1248,15 @@ void loadConfFile(AbstractConfig & config);
 // Used by the Settings constructor
 std::vector<Path> getUserConfigFiles();
 
-extern const std::string nixVersion;
+/**
+ * The version of Nix itself.
+ *
+ * This is not `const`, so that the Nix CLI can provide a more detailed version
+ * number including the git revision, without having to "re-compile" the entire
+ * set of Nix libraries to include that version, even when those libraries are
+ * not affected by the change.
+ */
+extern std::string nixVersion;
 
 extern const std::string determinateNixVersion;
 

src/libstore/legacy-ssh-store.cc

@@ -69,7 +69,10 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
         command.push_back("--store");
         command.push_back(remoteStore.get());
     }
-    conn->sshConn = master.startCommand(std::move(command));
+    conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs});
+    if (connPipeSize) {
+        conn->sshConn->trySetBufferSize(*connPipeSize);
+    }
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
 
@@ -100,19 +103,31 @@ std::string LegacySSHStore::getUri()
     return *uriSchemes().begin() + "://" + host;
 }
 
+std::map<StorePath, UnkeyedValidPathInfo> LegacySSHStore::queryPathInfosUncached(
+    const StorePathSet & paths)
+{
+    auto conn(connections->get());
+
+    /* No longer support missing NAR hash */
+    assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
+    debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths)));
+
+    auto infos = conn->queryPathInfos(*this, paths);
+
+    for (const auto & [_, info] : infos) {
+        if (info.narHash == Hash::dummy)
+            throw Error("NAR hash is now mandatory");
+    }
+
+    return infos;
+}
 
 void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto conn(connections->get());
-
-        /* No longer support missing NAR hash */
-        assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
-
-        debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
-
-        auto infos = conn->queryPathInfos(*this, {path});
+        auto infos = queryPathInfosUncached({path});
 
         switch (infos.size()) {
         case 0:
@@ -120,9 +135,6 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
         case 1: {
             auto & [path2, info] = *infos.begin();
 
-            if (info.narHash == Hash::dummy)
-                throw Error("NAR hash is now mandatory");
-
             assert(path == path2);
             return callback(std::make_shared<ValidPathInfo>(
                 std::move(path),
@@ -193,13 +205,19 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
 
 void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
 {
-    auto conn(connections->get());
-    conn->narFromPath(*this, path, [&](auto & source) {
+    narFromPath(path, [&](auto & source) {
         copyNAR(source, sink);
     });
 }
 
 
+void LegacySSHStore::narFromPath(const StorePath & path, std::function<void(Source &)> fun)
+{
+    auto conn(connections->get());
+    conn->narFromPath(*this, path, fun);
+}
+
+
 static ServeProto::BuildOptions buildSettings()
 {
     return {
@@ -223,6 +241,19 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
     return conn->getBuildDerivationResponse(*this);
 }
 
+std::function<BuildResult()> LegacySSHStore::buildDerivationAsync(
+    const StorePath & drvPath, const BasicDerivation & drv,
+    const ServeProto::BuildOptions & options)
+{
+    // Until we have C++23 std::move_only_function
+    auto conn = std::make_shared<Pool<Connection>::Handle>(connections->get());
+    (*conn)->putBuildDerivationRequest(*this, drvPath, drv, options);
+
+    return [this,conn]() -> BuildResult {
+        return (*conn)->getBuildDerivationResponse(*this);
+    };
+}
+
 
 void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
 {
@@ -294,6 +325,32 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
 }
 
 
+StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
+    bool lock, SubstituteFlag maybeSubstitute)
+{
+    auto conn(connections->get());
+    return conn->queryValidPaths(*this,
+        lock, paths, maybeSubstitute);
+}
+
+
+void LegacySSHStore::addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths)
+{
+    auto conn(connections->get());
+    conn->to << ServeProto::Command::ImportPaths;
+    try {
+        srcStore.exportPaths(paths, conn->to);
+    } catch (...) {
+        conn->good = false;
+        throw;
+    }
+    conn->to.flush();
+
+    if (readInt(conn->from) != 1)
+        throw Error("remote machine failed to import closure");
+}
+
+
 void LegacySSHStore::connect()
 {
     auto conn(connections->get());
@@ -307,6 +364,28 @@ unsigned int LegacySSHStore::getProtocol()
 }
 
 
+pid_t LegacySSHStore::getConnectionPid()
+{
+    auto conn(connections->get());
+#ifndef _WIN32
+    return conn->sshConn->sshPid;
+#else
+    // TODO: Implement
+    return 0;
+#endif
+}
+
+
+LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats()
+{
+    auto conn(connections->get());
+    return {
+        .bytesReceived = conn->from.read,
+        .bytesSent = conn->to.written,
+    };
+}
+
+
 /**
  * The legacy ssh protocol doesn't support checking for trusted-user.
  * Try using ssh-ng:// instead if you want to know.

src/libstore/legacy-ssh-store.hh

@@ -6,6 +6,7 @@
 #include "ssh.hh"
 #include "callback.hh"
 #include "pool.hh"
+#include "serve-protocol.hh"
 
 namespace nix {
 
@@ -24,6 +25,16 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
     const Setting<int> maxConnections{this, 1, "max-connections",
         "Maximum number of concurrent SSH connections."};
 
+    /**
+     * Hack for hydra
+     */
+    Strings extraSshArgs = {};
+
+    /**
+     * Exposed for hydra
+     */
+    std::optional<size_t> connPipeSize;
+
     const std::string name() override { return "SSH Store"; }
 
     static std::set<std::string> uriSchemes() { return {"ssh"}; }
@@ -60,11 +71,24 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
+    std::map<StorePath, UnkeyedValidPathInfo> queryPathInfosUncached(
+        const StorePathSet & paths);
+
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
+    /**
+     * Hands over the connection temporarily as source to the given
+     * function. The function must not consume beyond the NAR; it can
+     * not just blindly try to always read more bytes until it is
+     * cut-off.
+     *
+     * This is exposed for sake of Hydra.
+     */
+    void narFromPath(const StorePath & path, std::function<void(Source &)> fun);
+
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
@@ -93,6 +117,16 @@ public:
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
 
+    /**
+     * Note, the returned function must only be called once, or we'll
+     * try to read from the connection twice.
+     *
+     * @todo Use C++23 `std::move_only_function`.
+     */
+    std::function<BuildResult()> buildDerivationAsync(
+        const StorePath & drvPath, const BasicDerivation & drv,
+        const ServeProto::BuildOptions & options);
+
     void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
 
     void ensurePath(const StorePath & path) override
@@ -119,10 +153,36 @@ public:
     StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
+    /**
+     * Custom variation that atomically creates temp locks on the remote
+     * side.
+     *
+     * This exists to prevent a race where the remote host
+     * garbage-collects paths that are already there. Optionally, ask
+     * the remote host to substitute missing paths.
+     */
+    StorePathSet queryValidPaths(const StorePathSet & paths,
+        bool lock,
+        SubstituteFlag maybeSubstitute = NoSubstitute);
+
+    /**
+     * Just exists because this is exactly what Hydra was doing, and we
+     * don't yet want an algorithmic change.
+     */
+    void addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths);
+
     void connect() override;
 
     unsigned int getProtocol() override;
 
+    struct ConnectionStats {
+        size_t bytesReceived, bytesSent;
+    };
+
+    ConnectionStats getConnectionStats();
+
+    pid_t getConnectionPid();
+
     /**
      * The legacy ssh protocol doesn't support checking for trusted-user.
      * Try using ssh-ng:// instead if you want to know.

src/libstore/local-store.cc

@@ -136,7 +136,12 @@ LocalStore::LocalStore(
     for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
         createDirs(perUserDir);
         if (!readOnly) {
-            if (chmod(perUserDir.c_str(), 0755) == -1)
+            auto st = lstat(perUserDir);
+
+            // Skip chmod call if the directory already has the correct permissions (0755).
+            // This is to avoid failing when the executing user lacks permissions to change the directory's permissions
+            // even if it would be no-op.
+            if ((st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != 0755 && chmod(perUserDir.c_str(), 0755) == -1)
                 throw SysError("could not set permissions on '%s' to 755", perUserDir);
         }
     }

src/libstore/package.nix

@@ -71,14 +71,6 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags =
     [
       (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)

src/libstore/ssh.cc

@@ -240,4 +240,19 @@ Path SSHMaster::startMaster()
 
 #endif
 
+void SSHMaster::Connection::trySetBufferSize(size_t size)
+{
+#ifdef F_SETPIPE_SZ
+    /* This `fcntl` method of doing this takes a positive `int`. Check
+       and convert accordingly.
+
+       The function overall still takes `size_t` because this is more
+       portable for a platform-agnostic interface. */
+    assert(size <= INT_MAX);
+    int pipesize = size;
+    fcntl(in.get(), F_SETPIPE_SZ, pipesize);
+    fcntl(out.get(), F_SETPIPE_SZ, pipesize);
+#endif
+}
+
 }

src/libstore/ssh.hh

@@ -54,6 +54,18 @@ public:
         Pid sshPid;
 #endif
         AutoCloseFD out, in;
+
+        /**
+         * Try to set the buffer size in both directions to the
+         * designated amount, if possible. If not possible, does
+         * nothing.
+         *
+         * Current implementation is to use `fcntl` with `F_SETPIPE_SZ`,
+         * which is Linux-only. For this implementation, `size` must
+         * convertable to an `int`. In other words, it must be within
+         * `[0, INT_MAX]`.
+         */
+        void trySetBufferSize(size_t size);
     };
 
     /**

src/libstore/store-api.cc

@@ -230,18 +230,22 @@ void Store::addMultipleToStore(
 {
     std::atomic<size_t> nrDone{0};
     std::atomic<size_t> nrFailed{0};
-    std::atomic<uint64_t> bytesExpected{0};
     std::atomic<uint64_t> nrRunning{0};
 
     using PathWithInfo = std::pair<ValidPathInfo, std::unique_ptr<Source>>;
 
+    uint64_t bytesExpected = 0;
+
     std::map<StorePath, PathWithInfo *> infosMap;
     StorePathSet storePathsToAdd;
     for (auto & thingToAdd : pathsToCopy) {
+        bytesExpected += thingToAdd.first.narSize;
         infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd);
         storePathsToAdd.insert(thingToAdd.first.path);
     }
 
+    act.setExpected(actCopyPath, bytesExpected);
+
     auto showProgress = [&, nrTotal = pathsToCopy.size()]() {
         act.progress(nrDone, nrTotal, nrRunning, nrFailed);
     };
@@ -259,9 +263,6 @@ void Store::addMultipleToStore(
                 return StorePathSet();
             }
 
-            bytesExpected += info.narSize;
-            act.setExpected(actCopyPath, bytesExpected);
-
             return info.references;
         },