Merge commit '46d86e06ba' into progress-bar

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,7 @@
+**Release Notes**
+Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed.
+
+
+**Testing**
+
+If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).

doc/manual/src/SUMMARY.md.in

@@ -72,6 +72,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for
 $ uname
 Linux
 
-$ nix build \
+$ nix build --impure \
-  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
 [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
 

doc/manual/src/package-management/basic-package-mgmt.md

@@ -40,7 +40,7 @@ $ nix-channel --update
 > 
 > On NixOS, you’re automatically subscribed to a NixOS channel
 > corresponding to your NixOS major release (e.g.
-> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
 > to the Nixpkgs channel, except that it contains only Linux binaries
 > and is updated only if a set of regression tests succeed.
 

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,16 @@
+# Release 2.5 (2021-12-13)
+
+* The garbage collector no longer blocks new builds, so the message
+  `waiting for the big garbage collector lock...` is a thing of the
+  past.
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.
+
+* New built-in function: `builtins.groupBy`, with the same functionality as
+  Nixpkgs' `lib.groupBy`, but faster.
+
+* `nix repl` now has a `:log` command.

doc/manual/src/release-notes/rl-next.md

@@ -1,12 +1 @@
-# Release 2.5 (2021-XX-XX)
+# Release X.Y (202?-??-??)
-
-* Binary cache stores now have a setting `compression-level`.
-
-* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
-
-* Lists can now be compared lexicographically using the `<` operator.
-
-* New built-in function: `builtins.groupBy`, with the same functionality as
-  Nixpkgs' `lib.groupBy`, but faster.
-
-* `nix repl` now has a `:log` command.

src/libcmd/command.cc

@@ -73,8 +73,13 @@ ref<Store> EvalCommand::getEvalStore()
 
 ref<EvalState> EvalCommand::getEvalState()
 {
-    if (!evalState)
+    if (!evalState) evalState =
-        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+#if HAVE_BOEHMGC
+        std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+#else
+        std::make_shared<EvalState>(
+#endif
+            searchPath, getEvalStore(), getStore());
     return ref<EvalState>(evalState);
 }
 

src/libexpr/eval.cc

@@ -1656,7 +1656,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     bool first = !forceString;
     ValueType firstType = nString;
 
-    for (auto & i : *es) {
+    for (auto & [i_pos, i] : *es) {
         Value vTmp;
         i->eval(state, env, vTmp);
 
@@ -1677,19 +1677,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
+                throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == nFloat) {
             if (vTmp.type() == nInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
+                throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp));
         } else
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
-            s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
+            s << state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first);
 
         first = false;
     }

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ void ExprConcatStrings::show(std::ostream & str) const
     str << "(";
     for (auto & i : *es) {
         if (first) first = false; else str << " + ";
-        str << *i;
+        str << i.second;
     }
     str << ")";
 }
@@ -439,7 +439,7 @@ void ExprOpNot::bindVars(const StaticEnv & env)
 void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
     for (auto & i : *es)
-        i->bindVars(env);
+        i.second->bindVars(env);
 }
 
 void ExprPos::bindVars(const StaticEnv & env)

src/libexpr/nixexpr.hh

@@ -332,8 +332,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<Expr *> * es;
+    vector<std::pair<Pos, Expr *> > * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
+    ExprConcatStrings(const Pos & pos, bool forceString, vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -152,7 +152,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
-static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
+static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<std::pair<Pos, Expr *> > & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
 
@@ -162,7 +162,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     size_t minIndent = 1000000;
     size_t curIndent = 0;
-    for (auto & i : es) {
+    for (auto & [i_pos, i] : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
@@ -192,12 +192,12 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     }
 
     /* Strip spaces from each line. */
-    vector<Expr *> * es2 = new vector<Expr *>;
+    vector<std::pair<Pos, Expr *> > * es2 = new vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
-    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
+    for (vector<std::pair<Pos, Expr *> >::iterator i = es.begin(); i != es.end(); ++i, --n) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(i->second);
         if (!e) {
             atStartOfLine = false;
             curDropped = 0;
@@ -234,11 +234,11 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
                 s2 = string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(symbols.create(s2)));
+        es2->emplace_back(i->first, new ExprString(symbols.create(s2)));
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2);
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
 }
 
 
@@ -277,7 +277,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   char * path;
   char * uri;
   std::vector<nix::AttrName> * attrNames;
-  std::vector<nix::Expr *> * string_parts;
+  std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -364,7 +364,7 @@ expr_op
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
   | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
   | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
@@ -410,7 +410,7 @@ expr_simple
   }
   | path_start PATH_END { $$ = $1; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), $1);
+      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
@@ -448,13 +448,13 @@ string_parts
   ;
 
 string_parts_interpolated
-  : string_parts_interpolated STR { $$ = $1; $1->push_back($2); }
+  : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); }
+  | DOLLAR_CURLY expr '}' { $$ = new vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<Expr *>;
+      $$ = new vector<std::pair<Pos, Expr *> >;
-      $$->push_back($1);
+      $$->emplace_back(makeCurPos(@1, data), $1);
-      $$->push_back($3);
+      $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
 
@@ -473,9 +473,9 @@ path_start
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
-  | { $$ = new vector<Expr *>; }
+  | { $$ = new vector<std::pair<Pos, Expr *> >; }
   ;
 
 binds

src/libmain/shared.cc

@@ -145,10 +145,10 @@ static void preloadNSS() {
          *
          * All other platforms are unaffected.
          */
-        if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) {
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
-            printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend"));
+            warn("unable to load nss_dns backend");
-        }
+        // FIXME: get hosts entry from nsswitch.conf.
-        __nss_configure_lookup ("hosts", "dns");
+        __nss_configure_lookup("hosts", "files dns");
 #endif
     });
 }

src/libstore/build/local-derivation-goal.cc

@@ -1787,11 +1787,14 @@ void LocalDerivationGoal::runChild()
            i686-linux build on an x86_64-linux machine. */
         struct utsname utsbuf;
         uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
+        if ((drv->platform == "i686-linux"
-            (settings.thisSystem == "x86_64-linux" ||
+                && (settings.thisSystem == "x86_64-linux"
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+                    || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
+            || drv->platform == "armv7l-linux"
+            || drv->platform == "armv6l-linux")
+        {
             if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
+                throw SysError("cannot set 32-bit personality");
         }
 
         /* Impersonate a Linux 2.6 machine to get some determinism in

src/libstore/daemon.cc

@@ -956,7 +956,7 @@ void processConnection(
 
     Finally finally([&]() {
         _isInterrupted = false;
-        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
+        printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
     });
 
     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
@@ -989,6 +989,8 @@ void processConnection(
                 break;
             }
 
+            printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op);
+
             opCount++;
 
             try {

src/libstore/gc.cc

@@ -126,7 +126,17 @@ void LocalStore::addTempRoot(const StorePath & path)
             auto socketPath = stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
             state->fdRootsSocket = createUnixDomainSocket();
-            nix::connect(state->fdRootsSocket.get(), socketPath);
+            try {
+                nix::connect(state->fdRootsSocket.get(), socketPath);
+            } catch (SysError & e) {
+                /* The garbage collector may have exited, so we need to
+                   restart. */
+                if (e.errNo == ECONNREFUSED) {
+                    debug("GC socket connection refused");
+                    state->fdRootsSocket.close();
+                    goto restart;
+                }
+            }
         }
 
         try {
@@ -523,6 +533,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
                 if (!fdClient) continue;
 
+                debug("GC roots server accepted new client");
+
                 /* Process the connection in a separate thread. */
                 auto fdClient_ = fdClient.get();
                 std::thread clientThread([&, fdClient = std::move(fdClient)]() {
@@ -535,6 +547,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                         }
                     });
 
+                    /* On macOS, accepted sockets inherit the
+                       non-blocking flag from the server socket, so
+                       explicitly make it blocking. */
+                    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1)
+                        abort();
+
                     while (true) {
                         try {
                             auto path = readLine(fdClient.get());
@@ -559,7 +577,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                             } else
                                 printError("received garbage instead of a root from client");
                             writeFull(fdClient.get(), "1", false);
-                        } catch (Error &) { break; }
+                        } catch (Error & e) {
+                            debug("reading GC root from client: %s", e.msg());
+                            break;
+                        }
                     }
                 });
 

src/libstore/globals.hh

@@ -797,6 +797,15 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+
+    Setting<StringSet> ignoredAcls{
+        this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls",
+        R"(
+          A list of ACLs that should be ignored, normally Nix attempts to
+          remove all ACLs from files and directories in the Nix store, but
+          some ACLs like `security.selinux` or `system.nfs4_acl` can't be
+          removed even by root. Therefore it's best to just ignore them.
+        )"};
 #endif
 
     Setting<Strings> hashedMirrors{

src/libstore/local-store.cc

@@ -590,9 +590,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
             throw SysError("querying extended attributes of '%s'", path);
 
         for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
-            /* Ignore SELinux security labels since these cannot be
+            if (settings.ignoredAcls.get().count(eaName)) continue;
-               removed even by root. */
-            if (eaName == "security.selinux") continue;
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
                 throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
         }

src/libstore/store-api.cc

@@ -1079,7 +1079,7 @@ std::map<StorePath, StorePath> copyPaths(
                     nrFailed++;
                     if (!settings.keepGoing)
                         throw e;
-                    logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
+                    printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
                     showProgress();
                     return;
                 }

src/libutil/logging.hh

@@ -197,13 +197,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
 /* Print a string message if the current log level is at least the specified
    level. Note that this has to be implemented as a macro to ensure that the
    arguments are evaluated lazily. */
-#define printMsg(level, args...) \
+#define printMsgUsing(loggerParam, level, args...) \
     do { \
         auto __lvl = level; \
         if (__lvl <= nix::verbosity) { \
-            logger->log(__lvl, fmt(args)); \
+            loggerParam->log(__lvl, fmt(args)); \
         } \
     } while (0)
+#define printMsg(level, args...) printMsgUsing(logger, level, args)
 
 #define printError(args...) printMsg(lvlError, args)
 #define notice(args...) printMsg(lvlNotice, args)

src/libutil/tarfile.cc

@@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         else
             archive.check(r);
 
-        archive_entry_set_pathname(entry,
+        archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // Patch hardlink path
+        const char *original_hardlink = archive_entry_hardlink(entry);
+        if (original_hardlink) {
+            archive_entry_copy_hardlink(entry,
+                (destDir + "/" + original_hardlink).c_str());
+        }
+
         archive.check(archive_read_extract(archive.archive, entry, flags));
     }
 

src/nix/app.cc

@@ -83,11 +83,14 @@ UnresolvedApp Installable::toApp(EvalState & state)
         auto outPath = cursor->getAttr(state.sOutPath)->getString();
         auto outputName = cursor->getAttr(state.sOutputName)->getString();
         auto name = cursor->getAttr(state.sName)->getString();
+        auto aPname = cursor->maybeGetAttr("pname");
         auto aMeta = cursor->maybeGetAttr("meta");
         auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
         auto mainProgram =
             aMainProgram
             ? aMainProgram->getString()
+            : aPname
+            ? aPname->getString()
             : DrvName(name).name;
         auto program = outPath + "/bin/" + mainProgram;
         return UnresolvedApp { App {

src/nix/repl.cc

@@ -672,6 +672,8 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v)
 {
     if (displ >= envSize)
         throw Error("environment full; cannot add more variables");
+    if (auto oldVar = staticEnv.find(name); oldVar != staticEnv.vars.end())
+        staticEnv.vars.erase(oldVar);
     staticEnv.vars.emplace_back(name, displ);
     staticEnv.sort();
     env->values[displ++] = &v;

src/nix/run.md

@@ -43,10 +43,15 @@ program specified by the app definition.
 
 If *installable* evaluates to a derivation, it will try to execute the
 program `<out>/bin/<name>`, where *out* is the primary output store
-path of the derivation and *name* is the `meta.mainProgram` attribute
+path of the derivation, and *name* is the first of the following that
-of the derivation if it exists, and otherwise the name part of the
+exists:
-value of the `name` attribute of the derivation (e.g. if `name` is set
+
-to `hello-1.10`, it will run `$out/bin/hello`).
+* The `meta.mainProgram` attribute of the derivation.
+* The `pname` attribute of the derivation.
+* The name part of the value of the `name` attribute of the derivation.
+
+For instance, if `name` is set to `hello-1.10`, `nix run` will run
+`$out/bin/hello`.
 
 # Flake output attributes
 

tests/gc-non-blocking.sh

@@ -19,7 +19,7 @@ pid=$!
 
 sleep 2
 
-outPath=$(nix-build -o "$TEST_ROOT/result" -E "
+outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E "
   with import ./config.nix;
   mkDerivation {
     name = \"non-blocking\";

tests/repl.sh

@@ -1,6 +1,7 @@
 source common.sh
 
 replCmds="
+simple = 1
 simple = import ./simple.nix
 :b simple
 :log simple