Merge commit '971382cab0' into progress-bar

src/libcmd/installables.cc

@@ -291,6 +291,9 @@ void completeFlakeRefWithFragment(
 
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
+    if (!settings.isExperimentalFeatureEnabled(Xp::Flakes))
+        return;
+
     if (prefix == "")
         completions->add(".");
 

src/libexpr/flake/flake.cc

@@ -482,11 +482,16 @@ LockedFlake lockFlake(
                             }
                         }
 
+                        LockParent newParent {
+                            .path = inputPath,
+                            .absolute = false
+                        };
+
                         computeLocks(
                             mustRefetch
                             ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
                             : fakeInputs,
-                            childNode, inputPath, oldLock, parent, parentPath);
+                            childNode, inputPath, oldLock, newParent, parentPath);
 
                     } else {
                         /* We need to create a new lock file entry. So fetch

src/libexpr/primops.cc

@@ -2928,6 +2928,56 @@ static RegisterPrimOp primop_partition({
     .fun = prim_partition,
 });
 
+static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceFunction(*args[0], pos);
+    state.forceList(*args[1], pos);
+
+    ValueVectorMap attrs;
+
+    for (auto vElem : args[1]->listItems()) {
+        Value res;
+        state.callFunction(*args[0], *vElem, res, pos);
+        string name = state.forceStringNoCtx(res, pos);
+        Symbol sym = state.symbols.create(name);
+        auto vector = attrs.try_emplace(sym, ValueVector()).first;
+        vector->second.push_back(vElem);
+    }
+
+    state.mkAttrs(v, attrs.size());
+
+    for (auto & i : attrs) {
+        Value * list = state.allocAttr(v, i.first);
+        auto size = i.second.size();
+        state.mkList(*list, size);
+        memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size);
+    }
+}
+
+static RegisterPrimOp primop_groupBy({
+    .name = "__groupBy",
+    .args = {"f", "list"},
+    .doc = R"(
+      Groups elements of *list* together by the string returned from the
+      function *f* called on each element. It returns an attribute set
+      where each attribute value contains the elements of *list* that are
+      mapped to the same corresponding attribute name returned by *f*.
+
+      For example,
+
+      ```nix
+      builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"]
+      ```
+
+      evaluates to
+
+      ```nix
+      { b = [ "bar" "baz" ]; f = [ "foo" ]; }
+      ```
+    )",
+    .fun = prim_groupBy,
+});
+
 static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceFunction(*args[0], pos);
@@ -3732,7 +3782,7 @@ void EvalState::createBaseEnv()
                 .fun = primOp.fun,
                 .arity = std::max(primOp.args.size(), primOp.arity),
                 .name = symbols.create(primOp.name),
-                .args = std::move(primOp.args),
+                .args = primOp.args,
                 .doc = primOp.doc,
             });
 

src/libexpr/value.hh

@@ -425,9 +425,11 @@ void mkPath(Value & v, const char * s);
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
+typedef std::map<Symbol, ValueVector> ValueVectorMap;
 #endif
 
 

src/libfetchers/path.cc

@@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme
             // for security, ensure that if the parent is a store path, it's inside it
             if (store->isInStore(parent)) {
                 auto storePath = store->printStorePath(store->toStorePath(parent).first);
-                if (!isInDir(absPath, storePath))
+                if (!isDirOrInDir(absPath, storePath))
                     throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
             }
         } else

src/libmain/progress-bar.cc

@@ -20,7 +20,7 @@ ProgressBarSettings progressBarSettings;
 
 static GlobalConfig::Register rProgressBarSettings(&progressBarSettings);
 
-static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
+static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n)
 {
     assert(n < fields.size());
     assert(fields[n].type == Logger::Field::tString);

src/libmain/shared.cc

@@ -427,7 +427,7 @@ RunPager::RunPager()
     });
 
     pid.setKillSignal(SIGINT);
-
+    stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0);
     if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
         throw SysError("dupping stdout");
 }
@@ -438,7 +438,7 @@ RunPager::~RunPager()
     try {
         if (pid != -1) {
             std::cout.flush();
-            close(STDOUT_FILENO);
+            dup2(stdout, STDOUT_FILENO);
             pid.wait();
         }
     } catch (...) {

src/libmain/shared.hh

@@ -88,6 +88,7 @@ public:
 
 private:
     Pid pid;
+    int stdout;
 };
 
 extern volatile ::sig_atomic_t blockInt;

src/libstore/build/derivation-goal.cc

@@ -656,7 +656,7 @@ void DerivationGoal::tryLocalBuild() {
     throw Error(
         "unable to build with a primary store that isn't a local store; "
         "either pass a different '--store' or enable remote builds."
-        "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+        "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 }
 
 

src/libstore/build/substitution-goal.cc

@@ -140,6 +140,8 @@ void PathSubstitutionGoal::tryNext()
     {
         warn("substituter '%s' does not have a valid signature for path '%s'",
             sub->getUri(), worker.store.printStorePath(storePath));
+        warn("verify that your nix.conf contains a correct signature in 'trusted-public-keys' for %s",
+            sub->getUri());
         tryNext();
         return;
     }

src/libstore/build/worker.cc

@@ -291,11 +291,11 @@ void Worker::run(const Goals & _topGoals)
                 if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "
                             "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
                 else
                    throw Error("unable to start any build; remote machines may not have "
                             "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 
             }
             assert(!awake.empty());

src/libstore/daemon.cc

@@ -431,25 +431,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashType(hashAlgoRaw);
             }
 
-            StringSink saved;
-            TeeSource savedNARSource(from, saved);
-            RetrieveRegularNARSink savedRegular { saved };
-
-            if (method == FileIngestionMethod::Recursive) {
-                /* Get the entire NAR dump from the client and save it to
-                  a string so that we can pass it to
-                  addToStoreFromDump(). */
-                ParseSink sink; /* null sink; just parse the NAR */
-                parseDump(sink, savedNARSource);
-            } else
-                parseDump(savedRegular, from);
-
+            auto dumpSource = sinkToSource([&](Sink & saved) {
+                if (method == FileIngestionMethod::Recursive) {
+                    /* We parse the NAR dump through into `saved` unmodified,
+                       so why all this extra work? We still parse the NAR so
+                       that we aren't sending arbitrary data to `saved`
+                       unwittingly`, and we know when the NAR ends so we don't
+                       consume the rest of `from` and can't parse another
+                       command. (We don't trust `addToStoreFromDump` to not
+                       eagerly consume the entire stream it's given, past the
+                       length of the Nar. */
+                    TeeSource savedNARSource(from, saved);
+                    ParseSink sink; /* null sink; just parse the NAR */
+                    parseDump(sink, savedNARSource);
+                } else {
+                    /* Incrementally parse the NAR file, stripping the
+                       metadata, and streaming the sole file we expect into
+                       `saved`. */
+                    RetrieveRegularNARSink savedRegular { saved };
+                    parseDump(savedRegular, from);
+                    if (!savedRegular.regular) throw Error("regular file expected");
+                }
+            });
             logger->startWork();
-            if (!savedRegular.regular) throw Error("regular file expected");
-
-            // FIXME: try to stream directly from `from`.
-            StringSource dumpSource { *saved.s };
-            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);

src/libstore/local-store.cc

@@ -8,6 +8,7 @@
 #include "references.hh"
 #include "callback.hh"
 #include "topo-sort.hh"
+#include "finally.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1333,13 +1334,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
         dump.resize(oldSize + want);
         auto got = 0;
+        Finally cleanup([&]() {
+            dump.resize(oldSize + got);
+        });
         try {
             got = source.read(dump.data() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
         }
-        dump.resize(oldSize + got);
     }
 
     std::unique_ptr<AutoDelete> delTempDir;

src/libstore/names.cc

@@ -42,7 +42,7 @@ DrvName::~DrvName()
 { }
 
 
-bool DrvName::matches(DrvName & n)
+bool DrvName::matches(const DrvName & n)
 {
     if (name != "*") {
         if (!regex) {

src/libstore/names.hh

@@ -19,7 +19,7 @@ struct DrvName
     DrvName(std::string_view s);
     ~DrvName();
 
-    bool matches(DrvName & n);
+    bool matches(const DrvName & n);
 
 private:
     std::unique_ptr<Regex> regex;

src/libstore/remote-store.cc

@@ -684,6 +684,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
     Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
     auto conn(getConnection());
+
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) {
+        warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4");
+        try {
+            callback(nullptr);
+        } catch (...) { return callback.rethrow(); }
+    }
+
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();

src/libutil/util.cc

@@ -512,6 +512,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
     if (!fd)
         throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
     return {std::move(fd), tmpl};
 }
 

src/libutil/util.hh

@@ -11,6 +11,7 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include <atomic>
 #include <functional>
 #include <map>
 #include <sstream>

src/nix-build/nix-build.cc

@@ -359,6 +359,7 @@ static void main_nix_build(int argc, char * * argv)
            is not set, then build bashInteractive from
            <nixpkgs>. */
         auto shell = getEnv("NIX_BUILD_SHELL");
+        std::optional<StorePath> shellDrv;
 
         if (!shell) {
 
@@ -375,8 +376,7 @@ static void main_nix_build(int argc, char * * argv)
                 auto bashDrv = store->parseStorePath(drv->queryDrvPath());
                 pathsToBuild.push_back({bashDrv});
                 pathsToCopy.insert(bashDrv);
-
-                shell = drv->queryOutPath() + "/bin/bash";
+                shellDrv = bashDrv;
 
             } catch (Error & e) {
                 logError(e.info());
@@ -402,6 +402,11 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (shellDrv) {
+            auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value());
+            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
+        }
+
         if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             auto resolvedDrv = drv.tryResolve(*store);
             assert(resolvedDrv && "Successfully resolved the derivation");

src/nix-env/nix-env.cc

@@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors)
 }
 
 
+namespace {
+
+std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view prefix) {
+    constexpr std::size_t maxResults = 3;
+    std::set<std::string> result;
+    for (const auto & drvInfo : allElems) {
+        const auto drvName = DrvName { drvInfo.queryName() };
+        if (hasPrefix(drvName.name, prefix)) {
+            result.emplace(drvName.name);
+
+            if (result.size() >= maxResults) {
+                break;
+            }
+        }
+    }
+    return result;
+}
+
+struct Match
+{
+    DrvInfo drvInfo;
+    std::size_t index;
+
+    Match(DrvInfo drvInfo_, std::size_t index_)
+    : drvInfo{std::move(drvInfo_)}
+    , index{index_}
+    {}
+};
+
+/* If a selector matches multiple derivations
+   with the same name, pick the one matching the current
+   system.  If there are still multiple derivations, pick the
+   one with the highest priority.  If there are still multiple
+   derivations, pick the one with the highest version.
+   Finally, if there are still multiple derivations,
+   arbitrarily pick the first one. */
+std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches) {
+    /* Map from package names to derivations. */
+    std::map<std::string, Match> newest;
+    StringSet multiple;
+
+    for (auto & match : matches) {
+        auto & oneDrv = match.drvInfo;
+
+        const auto drvName = DrvName { oneDrv.queryName() };
+        long comparison = 1;
+
+        const auto itOther = newest.find(drvName.name);
+
+        if (itOther != newest.end()) {
+            auto & newestDrv = itOther->second.drvInfo;
+
+            comparison =
+                oneDrv.querySystem() == newestDrv.querySystem() ? 0 :
+                oneDrv.querySystem() == settings.thisSystem ? 1 :
+                newestDrv.querySystem() == settings.thisSystem ? -1 : 0;
+            if (comparison == 0)
+                comparison = comparePriorities(state, oneDrv, newestDrv);
+            if (comparison == 0)
+                comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version);
+        }
+
+        if (comparison > 0) {
+            newest.erase(drvName.name);
+            newest.emplace(drvName.name, match);
+            multiple.erase(drvName.fullName);
+        } else if (comparison == 0) {
+            multiple.insert(drvName.fullName);
+        }
+    }
+
+    matches.clear();
+    for (auto & [name, match] : newest) {
+        if (multiple.find(name) != multiple.end())
+            warn(
+                "there are multiple derivations named '%1%'; using the first one",
+                name);
+        matches.push_back(match);
+    }
+
+    return matches;
+}
+
+} // end namespace
+
 static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
     const Strings & args, bool newestOnly)
 {
@@ -232,79 +317,42 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
         selectors.emplace_back("*");
 
     DrvInfos elems;
-    set<unsigned int> done;
+    std::set<std::size_t> done;
 
-    for (auto & i : selectors) {
-        typedef list<std::pair<DrvInfo, unsigned int> > Matches;
-        Matches matches;
-        unsigned int n = 0;
-        for (DrvInfos::const_iterator j = allElems.begin();
-             j != allElems.end(); ++j, ++n)
-        {
-            DrvName drvName(j->queryName());
-            if (i.matches(drvName)) {
-                i.hits++;
-                matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n));
+    for (auto & selector : selectors) {
+        std::vector<Match> matches;
+        for (const auto & [index, drvInfo] : enumerate(allElems)) {
+            const auto drvName = DrvName { drvInfo.queryName() };
+            if (selector.matches(drvName)) {
+                ++selector.hits;
+                matches.emplace_back(drvInfo, index);
             }
         }
 
-        /* If `newestOnly', if a selector matches multiple derivations
-           with the same name, pick the one matching the current
-           system.  If there are still multiple derivations, pick the
-           one with the highest priority.  If there are still multiple
-           derivations, pick the one with the highest version.
-           Finally, if there are still multiple derivations,
-           arbitrarily pick the first one. */
         if (newestOnly) {
-
-            /* Map from package names to derivations. */
-            typedef map<string, std::pair<DrvInfo, unsigned int> > Newest;
-            Newest newest;
-            StringSet multiple;
-
-            for (auto & j : matches) {
-                DrvName drvName(j.first.queryName());
-                long d = 1;
-
-                Newest::iterator k = newest.find(drvName.name);
-
-                if (k != newest.end()) {
-                    d = j.first.querySystem() == k->second.first.querySystem() ? 0 :
-                        j.first.querySystem() == settings.thisSystem ? 1 :
-                        k->second.first.querySystem() == settings.thisSystem ? -1 : 0;
-                    if (d == 0)
-                        d = comparePriorities(state, j.first, k->second.first);
-                    if (d == 0)
-                        d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version);
-                }
-
-                if (d > 0) {
-                    newest.erase(drvName.name);
-                    newest.insert(Newest::value_type(drvName.name, j));
-                    multiple.erase(j.first.queryName());
-                } else if (d == 0) {
-                    multiple.insert(j.first.queryName());
-                }
-            }
-
-            matches.clear();
-            for (auto & j : newest) {
-                if (multiple.find(j.second.first.queryName()) != multiple.end())
-                    printInfo(
-                        "warning: there are multiple derivations named '%1%'; using the first one",
-                        j.second.first.queryName());
-                matches.push_back(j.second);
-            }
+            matches = pickNewestOnly(state, std::move(matches));
         }
 
         /* Insert only those elements in the final list that we
            haven't inserted before. */
-        for (auto & j : matches)
-            if (done.insert(j.second).second)
-                elems.push_back(j.first);
-    }
+        for (auto & match : matches)
+            if (done.insert(match.index).second)
+                elems.push_back(match.drvInfo);
 
-    checkSelectorUse(selectors);
+        if (selector.hits == 0 && selector.fullName != "*") {
+            const auto prefixHits = searchByPrefix(allElems, selector.name);
+            
+            if (prefixHits.empty()) {
+                throw Error("selector '%1%' matches no derivations", selector.fullName);    
+            } else {
+                std::string suggestionMessage = ", maybe you meant:";
+                for (const auto & drvName : prefixHits) {
+                    suggestionMessage += fmt("\n%s", drvName);
+                }
+                throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName);
+            }
+        }
+    }
 
     return elems;
 }

src/nix/registry.md

@@ -2,7 +2,7 @@ R""(
 
 # Description
 
-`nix flake` provides subcommands for managing *flake
+`nix registry` provides subcommands for managing *flake
 registries*. Flake registries are a convenience feature that allows
 you to refer to flakes using symbolic identifiers such as `nixpkgs`,
 rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You

src/nix/repl.cc

@@ -279,6 +279,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt)
     };
 
     setupSignals();
+    Finally resetTerminal([&]() { rl_deprep_terminal(); });
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
     restoreSignals();
@@ -356,6 +357,8 @@ StringSet NixRepl::completePrefix(string prefix)
             // Quietly ignore evaluation errors.
         } catch (UndefinedVarError & e) {
             // Quietly ignore undefined variable errors.
+        } catch (BadURL & e) {
+            // Quietly ignore BadURL flake-related errors.
         }
     }
 
@@ -427,7 +430,8 @@ bool NixRepl::processLine(string line)
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n"
              << "  :u <expr>     Build derivation, then start nix-shell\n"
-             << "  :doc <expr>   Show documentation of a builtin function\n";
+             << "  :doc <expr>   Show documentation of a builtin function\n"
+             << "  :log <expr>   Show logs for a derivation\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -497,7 +501,7 @@ bool NixRepl::processLine(string line)
         runNix("nix-shell", {state->store->printStorePath(drvPath)});
     }
 
-    else if (command == ":b" || command == ":i" || command == ":s") {
+    else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") {
         Value v;
         evalString(arg, v);
         StorePath drvPath = getDerivationPath(v);
@@ -511,6 +515,27 @@ bool NixRepl::processLine(string line)
                 logger->cout("  %s -> %s", outputName, state->store->printStorePath(outputPath));
         } else if (command == ":i") {
             runNix("nix-env", {"-i", drvPathRaw});
+        } else if (command == ":log") {
+            settings.readOnlyMode = true;
+            Finally roModeReset([&]() {
+                settings.readOnlyMode = false;
+            });
+            auto subs = getDefaultSubstituters();
+
+            subs.push_front(state->store);
+
+            bool foundLog = false;
+            RunPager pager;
+            for (auto & sub : subs) {
+                auto log = sub->getBuildLog(drvPath);
+                if (log) {
+                    printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri());
+                    logger->writeToStdout(*log);
+                    foundLog = true;
+                    break;
+                }
+            }
+            if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw);
         } else {
             runNix("nix-shell", {drvPathRaw});
         }

src/nix/repl.md

@@ -35,14 +35,17 @@ R""(
   nix-repl> emacs.drvPath
   "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv"
 
-  nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out"
+  nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out"
 
-  nix-repl> :b x
+  nix-repl> :b drv
   this derivation produced the following outputs:
     out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello
 
   nix-repl> builtins.readFile drv
   "Hello, world!\n"
+
+  nix-repl> :log drv
+  Hello, world!
   ```
 
 # Description