Merge branch 'master' into document-derivation-and-deriving-path

.mergify.yml

@@ -27,6 +27,7 @@ pull_request_rules:
         branches:
           - 2.18-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.19
@@ -37,6 +38,7 @@ pull_request_rules:
         branches:
           - 2.19-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.20
@@ -47,6 +49,7 @@ pull_request_rules:
         branches:
           - 2.20-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.21
@@ -57,6 +60,7 @@ pull_request_rules:
         branches:
           - 2.21-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.22
@@ -67,6 +71,7 @@ pull_request_rules:
         branches:
           - 2.22-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.23
@@ -77,6 +82,7 @@ pull_request_rules:
         branches:
           - 2.23-maintenance
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.24
@@ -87,6 +93,7 @@ pull_request_rules:
         branches:
           - "2.24-maintenance"
         labels:
+          - automatic backport
           - merge-queue
 
   - name: backport patches to 2.25
@@ -97,4 +104,5 @@ pull_request_rules:
         branches:
           - "2.25-maintenance"
         labels:
+          - automatic backport
           - merge-queue

doc/manual/source/installation/uninstall.md

@@ -160,6 +160,6 @@ which you may remove.
 To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run:
 
 ```console
-$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile
+rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile
 ```
 You might also want to manually remove references to Nix from your `~/.profile`.

packaging/dependencies.nix

@@ -75,7 +75,11 @@ let
       # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the
       # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10.
       # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable.
-      preConfigure = prevAttrs.preConfigure or "" + ''
+      preConfigure = prevAttrs.preConfigure or "" + lib.optionalString (
+        !stdenv.hostPlatform.isWindows
+        # build failure
+        && !stdenv.hostPlatform.isStatic
+      ) ''
         case "$mesonBuildType" in
         release|minsize) appendToVar mesonFlags "-Db_lto=true"  ;;
         *)               appendToVar mesonFlags "-Db_lto=false" ;;

packaging/everything.nix

@@ -61,6 +61,8 @@ let
       nix-store-c
       nix-util
       nix-util-c
+    ] ++ lib.optionals (!stdenv.hostPlatform.isStatic) [
+      # Currently fails in static build
       nix-perl-bindings
     ];
     installPhase = ''
@@ -131,6 +133,8 @@ in
     # (checkInputs must be empty paths??)
     (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out")
   ] ++
+    lib.optionals (!stdenv.hostPlatform.isStatic) (
+    # Perl currently fails in static build
     (if stdenv.buildPlatform.canExecute stdenv.hostPlatform
     then [
       # TODO: add perl.tests
@@ -138,7 +142,7 @@ in
     ]
     else [
       nix-perl-bindings
-    ]);
+    ]));
   installCheckInputs = [
     nix-functional-tests
   ];

src/libfetchers/git.cc

@@ -331,7 +331,7 @@ struct GitInputScheme : InputScheme
 
         auto result = runProgram(RunOptions {
             .program = "git",
-            .args = {"-C", *repoPath, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
+            .args = {"-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
         });
         auto exitCode =
 #ifndef WIN32 // TODO abstract over exit status handling on Windows
@@ -344,7 +344,7 @@ struct GitInputScheme : InputScheme
         if (exitCode != 0) {
             // The path is not `.gitignore`d, we can add the file.
             runProgram("git", true,
-                { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+                { "-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
 
             if (commitMsg) {
@@ -352,7 +352,7 @@ struct GitInputScheme : InputScheme
                 logger->pause();
                 Finally restoreLogger([]() { logger->resume(); });
                 runProgram("git", true,
-                    { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
+                    { "-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
                     *commitMsg);
             }
         }
@@ -470,7 +470,7 @@ struct GitInputScheme : InputScheme
         return repoInfo;
     }
 
-    uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
+    uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
 
@@ -486,7 +486,7 @@ struct GitInputScheme : InputScheme
         return lastModified;
     }
 
-    uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
+    uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
 
@@ -557,7 +557,7 @@ struct GitInputScheme : InputScheme
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
 
-        Path repoDir;
+        std::filesystem::path repoDir;
 
         if (auto repoPath = repoInfo.getPath()) {
             repoDir = *repoPath;
@@ -565,22 +565,22 @@ struct GitInputScheme : InputScheme
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
         } else {
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
-            Path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input));
+            std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input));
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
-            createDirs(dirOf(cacheDir));
+            std::filesystem::create_directories(cacheDir.parent_path());
-            PathLocks cacheDirLock({cacheDir});
+            PathLocks cacheDirLock({cacheDir.string()});
 
             auto repo = GitRepo::openRepo(cacheDir, true, true);
 
             // We need to set the origin so resolving submodule URLs works
             repo->setRemote("origin", repoUrl.to_string());
 
-            Path localRefFile =
+            auto localRefFile =
                 ref.compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + ref
+                ? cacheDir / ref
-                : cacheDir + "/refs/heads/" + ref;
+                : cacheDir / "refs/heads" / ref;
 
             bool doFetch;
             time_t now = time(0);
@@ -596,7 +596,7 @@ struct GitInputScheme : InputScheme
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
                        git fetch to update the local ref to the remote ref. */
                     struct stat st;
-                    doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+                    doFetch = stat(localRefFile.string().c_str(), &st) != 0 ||
                         !isCacheFileWithinTtl(now, st);
                 }
             }
@@ -616,7 +616,7 @@ struct GitInputScheme : InputScheme
 
                     repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
                 } catch (Error & e) {
-                    if (!pathExists(localRefFile)) throw;
+                    if (!std::filesystem::exists(localRefFile)) throw;
                     logError(e.info());
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.locationToArg());
                 }
@@ -850,7 +850,7 @@ struct GitInputScheme : InputScheme
                 for (auto & file : repoInfo.workdirInfo.dirtyFiles) {
                     writeString("modified:", hashSink);
                     writeString(file.abs(), hashSink);
-                    dumpPath(*repoPath / file.rel(), hashSink);
+                    dumpPath((*repoPath / file.rel()).string(), hashSink);
                 }
                 for (auto & file : repoInfo.workdirInfo.deletedFiles) {
                     writeString("deleted:", hashSink);

src/libflake/flake/flake.cc

@@ -783,7 +783,7 @@ LockedFlake lockFlake(
                             auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
                             auto outputLockFilePath = *sourcePath / relPath;
 
-                            bool lockFileExists = pathExists(outputLockFilePath);
+                            bool lockFileExists = fs::symlink_exists(outputLockFilePath);
 
                             auto s = chomp(diff);
                             if (lockFileExists) {

src/libstore/remote-store.cc

@@ -534,14 +534,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 void RemoteStore::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
     auto source = sinkToSource([&](Sink & sink) {
         sink << pathsToCopy.size();
-        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+        // Reverse, so we can release memory at the original start
+        std::reverse(pathsToCopy.begin(), pathsToCopy.end());
+        while (!pathsToCopy.empty()) {
+            auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {
                      .to = sink,
@@ -549,6 +552,7 @@ void RemoteStore::addMultipleToStore(
                  },
                  pathInfo);
             pathSource->drainInto(sink);
+            pathsToCopy.pop_back();
         }
     });
 

src/libstore/remote-store.hh

@@ -102,7 +102,7 @@ public:
         CheckSigsFlag checkSigs) override;
 
     void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;

src/libstore/store-api.cc

@@ -223,7 +223,7 @@ StorePath Store::addToStore(
 }
 
 void Store::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -1138,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
         pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
     }
 
-    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
+    dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
 
     return pathsMap;
 }

src/libstore/store-api.hh

@@ -425,7 +425,7 @@ public:
         CheckSigsFlag checkSigs = CheckSigs);
 
     virtual void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);

src/libutil/posix-source-accessor.cc

@@ -124,7 +124,9 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
             S_ISLNK(st->st_mode) ? tSymlink :
             S_ISCHR(st->st_mode) ? tChar :
             S_ISBLK(st->st_mode) ? tBlock :
+#ifdef S_ISSOCK
             S_ISSOCK(st->st_mode) ? tSocket :
+#endif
             S_ISFIFO(st->st_mode) ? tFifo :
             tUnknown,
         .fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,

src/libutil/thread-pool.hh

@@ -150,8 +150,16 @@ void processGraph(
         }
     };
 
-    for (auto & node : nodes)
+    for (auto & node : nodes) {
-        pool.enqueue(std::bind(worker, std::ref(node)));
+        try {
+            pool.enqueue(std::bind(worker, std::ref(node)));
+        } catch (ThreadPoolShutDown &) {
+            /* Stop if the thread pool is shutting down. It means a
+               previous work item threw an exception, so process()
+               below will rethrow it. */
+            break;
+        }
+    }
 
     pool.process();
 

src/nix/upgrade-nix.cc

@@ -97,7 +97,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
             // FIXME: don't call an external process.
             runProgram(getNixBin("nix-env").string(), false,
-                {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"});
+                {"--profile", profileDir.string(), "-i", store->printStorePath(storePath), "--no-sandbox"});
         }
 
         printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version);
@@ -120,7 +120,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         // Resolve profile to /nix/var/nix/profiles/<name> link.
         while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir))
-            profileDir = readLink(profileDir);
+            profileDir = readLink(profileDir.string());
 
         printInfo("found profile %s", profileDir);
 

tests/nixos/git-submodules.nix

@@ -39,11 +39,13 @@
       client.succeed("chmod 600 /root/.ssh/id_ed25519")
 
       # Install the SSH key on the builders.
-      client.wait_for_unit("network.target")
+      client.wait_for_unit("network-online.target")
 
       remote.succeed("mkdir -p -m 700 /root/.ssh")
       remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
       remote.wait_for_unit("sshd")
+      remote.wait_for_unit("multi-user.target")
+      remote.wait_for_unit("network-online.target")
       client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'")
 
       remote.succeed("""

tests/nixos/nix-copy-closure.nix

@@ -48,7 +48,10 @@ in {
     server.succeed("mkdir -m 700 /root/.ssh")
     server.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
     server.wait_for_unit("sshd")
-    client.wait_for_unit("network.target")
+    server.wait_for_unit("multi-user.target")
+    server.wait_for_unit("network-online.target")
+
+    client.wait_for_unit("network-online.target")
     client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'")
 
     # Copy the closure of package A from the client to the server.

tests/nixos/nix-copy.nix

@@ -56,7 +56,10 @@ in {
     start_all()
 
     server.wait_for_unit("sshd")
-    client.wait_for_unit("network.target")
+    server.wait_for_unit("multi-user.target")
+    server.wait_for_unit("network-online.target")
+
+    client.wait_for_unit("network-online.target")
     client.wait_for_unit("getty@tty1.service")
     # Either the prompt: ]#
     # or an OCR misreading of it: 1#

tests/nixos/remote-builds-ssh-ng.nix

@@ -89,10 +89,13 @@ in
       client.succeed("chmod 600 /root/.ssh/id_ed25519")
 
       # Install the SSH key on the builder.
-      client.wait_for_unit("network.target")
+      client.wait_for_unit("network-online.target")
       builder.succeed("mkdir -p -m 700 /root/.ssh")
       builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
       builder.wait_for_unit("sshd")
+      builder.wait_for_unit("multi-user.target")
+      builder.wait_for_unit("network-online.target")
+
       client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'")
 
       # Perform a build

tests/nixos/remote-builds.nix

@@ -112,11 +112,12 @@ in
       client.succeed("chmod 600 /root/.ssh/id_ed25519")
 
       # Install the SSH key on the builders.
-      client.wait_for_unit("network.target")
+      client.wait_for_unit("network-online.target")
       for builder in [builder1, builder2]:
         builder.succeed("mkdir -p -m 700 /root/.ssh")
         builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys")
         builder.wait_for_unit("sshd")
+        builder.wait_for_unit("network-online.target")
         # Make sure the builder can handle our login correctly
         builder.wait_for_unit("multi-user.target")
         # Make sure there's no funny business on the client either