From fcb8af550f5fca37458da0d9042a2b59523eb304 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 15 Oct 2021 16:25:49 +0200 Subject: [PATCH 01/85] Restore parent mount namespace in restoreProcessContext This ensures any started processes can't write to /nix/store (except during builds). This partially reverts 01d07b1e, which happened because of #2646. The problem was only happening after nix downloads anything, causing me to suspect the download thread. The problem turns out to be: "A process can't join a new mount namespace if it is sharing filesystem-related attributes with another process", in this case this process is the curl thread. Ideally, we might kill it before spawning the shell process, but it's inside a static variable in the getFileTransfer() function. So instead, stop it from sharing FS state using unshare(). A strategy such as the one from #5057 (single-threaded chroot helper binary) is also very much on the table. Fixes #4337. --- src/libstore/filetransfer.cc | 8 ++++++++ src/libstore/local-store.cc | 1 + src/libutil/util.cc | 28 ++++++++++++++++++++++++++-- src/libutil/util.hh | 10 +++++++++- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 37e17b397..4621a8217 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); +#ifdef __linux__ + /* Cause this thread to not share any FS attributes with the main thread, + because this causes setns() in restoreMountNamespace() to fail. + Ideally, this would happen in the std::thread() constructor. */ + if (unshare(CLONE_FS) != 0) + throw SysError("unsharing filesystem state in download thread"); +#endif + std::map> items; bool quit = false; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5b2490472..3440c3150 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -495,6 +495,7 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { + saveMountNamespace(); if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 563a72c12..a5e961c1e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1629,10 +1629,34 @@ void setStackSize(size_t stackSize) } #endif } +static AutoCloseFD fdSavedMountNamespace; -void restoreProcessContext() +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); +#endif +} + +void restoreProcessContext(bool restoreMounts) { restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } restoreAffinity(); @@ -1766,7 +1790,7 @@ void commonChildInit(Pipe & logPipe) logger = makeSimpleLogger(); const static string pathNullDevice = "/dev/null"; - restoreProcessContext(); + restoreProcessContext(false); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29232453f..ef3430689 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -300,7 +300,15 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal masks, stack size, CPU affinity). */ -void restoreProcessContext(); +void restoreProcessContext(bool restoreMounts = true); + +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); class ExecError : public Error From b9234142f5e3c69f3372f15ab2e6df19c7bf6b64 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Sat, 23 Oct 2021 21:30:51 +0300 Subject: [PATCH 02/85] addToStore, addToStoreFromDump: add references argument Allow to pass a set of references to be added as info to the added paths. --- src/libstore/binary-cache-store.cc | 6 ++++-- src/libstore/binary-cache-store.hh | 4 ++-- src/libstore/build/local-derivation-goal.cc | 8 +++++--- src/libstore/daemon.cc | 6 +++--- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 3 ++- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 3 +-- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 5 +++-- 11 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8fce94264..280f1d4b5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -308,7 +308,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); @@ -318,6 +318,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam nar.first, }; info.narSize = nar.second; + info.references = references; return info; })->path; } @@ -385,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, StorePathSet references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -408,6 +409,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath nar.first, }; info.narSize = nar.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = h, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 723f2e805..d8f077db2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -97,11 +97,11 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references ) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + PathFilter & filter, RepairFlag repair, StorePathSet references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8d245f84a..694c408f3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1179,7 +1179,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, + StorePathSet references = StorePathSet()) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1198,9 +1199,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo } StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + StorePathSet references = StorePathSet()) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); return path; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 164a9b2be..fa6606ba4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -402,9 +402,9 @@ static void performOp(TunnelLogger * logger, ref store, return store->queryPathInfo(path); }, [&](FixedOutputHashMethod & fohm) { - if (!refs.empty()) - throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + // if (!refs.empty()) + // throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 814960bb5..433ddc8e1 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override + PathFilter & filter, RepairFlag repair, StorePathSet references) override { unsupported("addToStore"); } StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5b2490472..85f1de0b0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1302,7 +1302,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1405,6 +1405,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, ValidPathInfo info { dstPath, narHash.first }; info.narSize = narHash.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..7a7329dd8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index fa5ea8af7..632dfcf21 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -578,9 +578,8 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) + FileIngestionMethod method, HashType hashType, RepairFlag repair, StorePathSet references) { - StorePathSet references; return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ac1eaa19e..31eb1871a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -73,7 +73,7 @@ public: /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, StorePathSet references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b5ff3dccf..84d6db297 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -237,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, StorePathSet references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -246,7 +246,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, else readFile(srcPath, sink); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); + return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 54471bdf2..c4fdb176a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,7 +453,7 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, StorePathSet references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -469,7 +469,8 @@ public: `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + StorePathSet references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is From ba81e871b2b3a034f3e2eaa5242c98e2f253cdd5 Mon Sep 17 00:00:00 2001 From: Shay Bergmann Date: Mon, 25 Oct 2021 21:13:35 +0000 Subject: [PATCH 03/85] toJSON: report error position for fancier output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given flake: ```nix { description = "nix json error provenance"; inputs = {}; outputs = { self }: { jsonFunction = _: "function"; json = builtins.toJSON (_: "function"); }; } ``` - Before: ```console ❯ nix eval --json .#jsonFunction error: cannot convert a function to JSON ``` - After: ```console ❯ nix eval --json .#jsonFunction error: cannot convert a function to JSON at /nix/store/b7imf1c2j4jnkg3ys7fsfbj02s5j0i4f-source/testflake/flake.nix:4:5: 3| outputs = { self }: { 4| jsonFunction = _: "function"; | ^ 5| json = builtins.toJSON (_: "function"); ``` --- src/libexpr/primops.cc | 4 ++-- src/libexpr/value-to-json.cc | 24 +++++++++++++++--------- src/libexpr/value-to-json.hh | 4 ++-- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/eval.cc | 2 +- 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4e0eda7f3..aac741f90 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1008,7 +1008,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, *i->value, pos, placeholder, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1687,7 +1687,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu { std::ostringstream out; PathSet context; - printValueAsJSON(state, true, *args[0], out, context); + printValueAsJSON(state, true, *args[0], pos, out, context); mkString(v, out.str(), context); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index bfea24d40..7c28754f5 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,11 +10,11 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -54,10 +54,10 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); + printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); } } else - printValueAsJSON(state, strict, *i->value, out, context); + printValueAsJSON(state, strict, *i->value, *i->pos, out, context); break; } @@ -65,7 +65,7 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + printValueAsJSON(state, strict, *v.listElems()[n], noPos, placeholder, context); } break; } @@ -79,18 +79,24 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError("cannot convert %1% to JSON", showType(v)); + throw TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = pos + }); case nFunction: - throw TypeError("cannot convert %1% to JSON", showType(v)); + throw TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = pos + }); } } void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, const Pos & pos, std::ostream & str, PathSet & context) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); + printValueAsJSON(state, strict, v, pos, out, context); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 67fed6487..c2f797b29 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); + Value & v, const Pos & pos, std::ostream & str, PathSet & context); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a86f55f84..4056d973d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -879,7 +879,7 @@ static void queryJSON(Globals & globals, vector & elems) placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 25d0fa3ba..ac48682c7 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,7 +52,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, std::cout, context); + printValueAsJSON(state, strict, vRes, noPos, std::cout, context); else { if (strict) state.forceValueDeep(vRes); std::cout << vRes << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 65d61e005..c7517cf79 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -112,7 +112,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context); } else { From 8919b81dad58af365919bcf3ded4117e973344d3 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 26 Oct 2021 20:02:37 +0700 Subject: [PATCH 04/85] Support building flakes from a Git repo url with submodules query parameter --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8468d2afc..a3f4e42a3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -51,7 +51,7 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow") + else if (name == "shallow" || name == "submodules") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); From 769de259f0060d55ec8317c109cf12aa4003f7fe Mon Sep 17 00:00:00 2001 From: Shay Bergmann Date: Tue, 26 Oct 2021 14:43:15 +0000 Subject: [PATCH 05/85] toJSON: pass pos in case of a list as well --- src/libexpr/value-to-json.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 7c28754f5..1c3849593 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -65,7 +65,7 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], noPos, placeholder, context); + printValueAsJSON(state, strict, *v.listElems()[n], pos, placeholder, context); } break; } From 465a167c4347cd208f39670bad349fc61bb42d3d Mon Sep 17 00:00:00 2001 From: Shay Bergmann Date: Wed, 27 Oct 2021 19:01:32 +0000 Subject: [PATCH 06/85] nix-instantiate: pass pos in the `--eval --json` code path --- src/nix-instantiate/nix-instantiate.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index ac48682c7..fbc697864 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,7 +52,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, noPos, std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); else { if (strict) state.forceValueDeep(vRes); std::cout << vRes << std::endl; From a50c027ece2dc853109c7d2c2f10dd6581e2554d Mon Sep 17 00:00:00 2001 From: Shay Bergmann Date: Wed, 27 Oct 2021 19:48:48 +0000 Subject: [PATCH 07/85] toJSON: improve pos accuracy, add trace --- src/libexpr/value-to-json.cc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 1c3849593..4d642c720 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { out.write(*maybeString); break; @@ -79,16 +79,12 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError({ - .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = pos - }); - case nFunction: - throw TypeError({ + auto e = TypeError({ .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = pos + .errPos = v.determinePos(pos) }); + throw e.addTrace(pos, hintfmt("message for the trace")); } } From b459a3e85600e1d3a326278a823ce0bdf71fcc21 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Sat, 30 Oct 2021 12:25:59 +0200 Subject: [PATCH 08/85] git: extend cache dir lock over all mutating operations --- src/libfetchers/git.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8468d2afc..a96642195 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -324,17 +324,13 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; - Path cacheDirLock = cacheDir + ".lock"; createDirs(dirOf(cacheDir)); - AutoCloseFD lock = openLockFile(cacheDirLock, true); - lockFile(lock.get(), ltWrite, true); + PathLocks cacheDirLock({cacheDir + ".lock"}); if (!pathExists(cacheDir)) { runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - deleteLockFile(cacheDirLock, lock.get()); - Path localRefFile = input.getRef()->compare(0, 5, "refs/") == 0 ? cacheDir + "/" + *input.getRef() @@ -399,6 +395,8 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + + // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; From 14fcf1727759f8eaf88a805cbe2251fbe9920edb Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Wed, 3 Nov 2021 14:10:16 +0200 Subject: [PATCH 09/85] libstore: Use unix-dotfile vfs if useSQLiteWAL is false --- src/libstore/sqlite.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 447b4179b..1d6baf02d 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path, bool create) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) From 3f070cc417ab71cf8b20d5a9db62ff515d209846 Mon Sep 17 00:00:00 2001 From: Christopher League Date: Wed, 3 Nov 2021 09:25:27 -0400 Subject: [PATCH 10/85] In checkOverlay, accept underscored names for final/prev args. Resolves #4416. --- src/nix/flake.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 68bb76742..5855348ac 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -346,10 +346,14 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() || v.lambda.fun->hasFormals() || + (std::string(v.lambda.fun->arg) != "final" && + std::string(v.lambda.fun->arg) != "_final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); - if (!body || body->hasFormals() || std::string(body->arg) != "prev") + if (!body || body->hasFormals() || + (std::string(body->arg) != "prev" && + std::string(body->arg) != "_prev")) throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. From c34cc5e488b7baf7dcc1944c72fdceaa6affb55a Mon Sep 17 00:00:00 2001 From: Dimitris Apostolou Date: Wed, 3 Nov 2021 18:11:20 +0200 Subject: [PATCH 11/85] Fix typos --- doc/manual/src/command-ref/nix-env.md | 2 +- doc/manual/src/command-ref/nix-store.md | 2 +- doc/manual/src/contributing/cli-guideline.md | 22 +++++++++---------- .../src/expressions/advanced-attributes.md | 2 +- .../src/expressions/expression-syntax.md | 2 +- doc/manual/src/expressions/language-values.md | 2 +- doc/manual/src/introduction.md | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 9138fa05a..5217510d7 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -401,7 +401,7 @@ of a derivation `x` by looking at their respective `name` attributes. The names (e.g., `gcc-3.3.1` are split into two parts: the package name (`gcc`), and the version (`3.3.1`). The version part starts after the first dash not followed by a letter. `x` is considered an upgrade of `y` -if their package names match, and the version of `y` is higher that that +if their package names match, and the version of `y` is higher than that of `x`. The versions are compared by splitting them into contiguous components diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 7a131dc02..26292f1bb 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -125,7 +125,7 @@ Special exit codes: - `104`\ Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducable. + resulting output is not binary reproducible. With the `--keep-going` flag it's possible for multiple failures to occur, in this case the 1xx status codes are or combined using binary diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 0132867c8..69bc45691 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -3,7 +3,7 @@ ## Goals Purpose of this document is to provide a clear direction to **help design -delightful command line** experience. This document contain guidelines to +delightful command line** experience. This document contains guidelines to follow to ensure a consistent and approachable user experience. ## Overview @@ -115,7 +115,7 @@ The rules are: - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or `nix help`). -- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show +- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show a summary** of most common use cases. Summary is presented on the STDOUT without any use of PAGER. - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the @@ -230,8 +230,8 @@ Now **Learn** part of the output is where you educate users. You should only show it when you know that a build will take some time and not annoy users of the builds that take only few seconds. -Every feature like this should go though a intensive review and testing to -collect as much a feedback as possible and to fine tune every little detail. If +Every feature like this should go through an intensive review and testing to +collect as much feedback as possible and to fine tune every little detail. If done right this can be an awesome features beginners and advance users will love, but if not done perfectly it will annoy users and leave bad impression. @@ -272,11 +272,11 @@ not know which `ARGUMENTS` and `OPTIONS` are required or which values are possible for those options. In cases, the user might not provide the input or they provide wrong input, -rather then show the error, prompt a user with an option to find and select +rather than show the error, prompt a user with an option to find and select correct input (see examples). Prompting is of course not required when TTY is not attached to STDIN. This -would mean that scripts wont need to handle prompt, but rather handle errors. +would mean that scripts won't need to handle prompt, but rather handle errors. A place to use prompt and provide user with interactive select @@ -300,7 +300,7 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ The following substitutors will be used to in `my-project`: - https://cache.example.org @@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org # Output -Terminal output can be quite limiting in many ways. Which should forces us to +Terminal output can be quite limiting in many ways. Which should force us to think about the experience even more. As with every design the output is a compromise between being terse and being verbose, between showing help to beginners and annoying advance users. For this it is important that we know what are the priorities. Nix command line should be first and foremost written with beginners in mind. -But users wont stay beginners for long and what was once useful might quickly +But users won't stay beginners for long and what was once useful might quickly become annoying. There is no golden rule that we can give in this guideline that would make it easier how to draw a line and find best compromise. @@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool. ### Progress -For longer running commands we should provide and overview of the progress. +For longer running commands we should provide and overview the progress. This is shown best in `nix build` example: ```shell @@ -553,7 +553,7 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ The following substitutors will be used to in `my-project`: - https://cache.example.org diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md index 5b208df67..000595815 100644 --- a/doc/manual/src/expressions/advanced-attributes.md +++ b/doc/manual/src/expressions/advanced-attributes.md @@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes. - `preferLocalBuild`\ If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivaton will be built locally instead of forwarded + possible, the derivation will be built locally instead of forwarded to a remote machine. This is appropriate for trivial builders where the cost of doing a download or remote build would exceed the cost of building locally. diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md index 2a1306e32..6b93e692c 100644 --- a/doc/manual/src/expressions/expression-syntax.md +++ b/doc/manual/src/expressions/expression-syntax.md @@ -26,7 +26,7 @@ elements (referenced from the figure by number): called with three arguments: `stdenv`, `fetchurl`, and `perl`. They are needed to build Hello, but we don't know how to build them here; that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages packages; it provides a + is used by almost all Nix Packages; it provides a “standard” environment consisting of the things you would expect in a basic Unix environment: a C/C++ compiler (GCC, to be precise), the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 28fa23b58..75ae9f2eb 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -64,7 +64,7 @@ Nix has the following basic data types: the start of each line. To be precise, it strips from each line a number of spaces equal to the minimal indentation of the string as a whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two space, while the third + the first and second line are indented two spaces, while the third line is indented four spaces. Thus, two spaces are stripped from each line, so the resulting string is diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index d68445c95..93ec502c1 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -127,7 +127,7 @@ $ nix-env --install firefox _could_ cause quite a bit of build activity, as not only Firefox but also all its dependencies (all the way up to the C library and the -compiler) would have to built, at least if they are not already in the +compiler) would have to be built, at least if they are not already in the Nix store. This is a _source deployment model_. For most users, building from source is not very pleasant as it takes far too long. However, Nix can automatically skip building from source and instead From 6f291ed718868a926c6c1aacac12359a78c60da7 Mon Sep 17 00:00:00 2001 From: Moritz Hedtke Date: Tue, 2 Nov 2021 15:28:01 +0100 Subject: [PATCH 12/85] Fix leaking pthread_attr_t pthread_attr_destroy was not called. --- boehmgc-coroutine-sp-fallback.diff | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index fa8dd0325..e659bf470 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 1cee6a0b..46c3acd9 100644 +index 4b2c429..1fb4c52 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c -@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void) +@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) struct GC_traced_stack_sect_s *traced_stack_sect; pthread_t self = pthread_self(); word total_size = 0; @@ -11,7 +11,7 @@ index 1cee6a0b..46c3acd9 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void) +@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ @@ -22,6 +22,9 @@ index 1cee6a0b..46c3acd9 100644 + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { + ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); + } ++ if (pthread_attr_destroy(&pattr)) { ++ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); ++ } + // When a thread goes into a coroutine, we lose its original sp until + // control flow returns to the thread. + // While in the coroutine, the sp points outside the thread stack, From 7d56174c1e0f50a06e897f83b3c1b97ac87dbf5a Mon Sep 17 00:00:00 2001 From: Jan Van Bruggen Date: Wed, 3 Nov 2021 18:42:32 -0600 Subject: [PATCH 13/85] Reword "we"s to "I"s for consistency This script uses multiple forms of the first-person POV: 1. "We" to refer to the Nix team (https://github.com/NixOS/nix/blob/1e7c796e66a692cd097b155bd0a60fedf20d554f/scripts/install-multi-user.sh#L72) 2. "We" to refer to the combination of the installation script & the user/executor (https://github.com/NixOS/nix/blob/1e7c796e66a692cd097b155bd0a60fedf20d554f/scripts/install-multi-user.sh#L710) 3. "We" to refer to the installation script alone (https://github.com/NixOS/nix/blob/1e7c796e66a692cd097b155bd0a60fedf20d554f/scripts/install-multi-user.sh#L602) 4. "I" to refer to the installation script alone (https://github.com/NixOS/nix/blob/1e7c796e66a692cd097b155bd0a60fedf20d554f/scripts/install-multi-user.sh#L200) Since I prefer POV 4 to POV 3, this changes all instances of POV 3 to POV 4. --- scripts/install-multi-user.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 513127a62..7d1cb8c5a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -599,7 +599,7 @@ manager. This will happen in a few stages: 1. Make sure your computer doesn't already have Nix. If it does, I will show you instructions on how to clean up your old install. -2. Show you what we are going to install and where. Then we will ask +2. Show you what I am going to install and where. Then I will ask if you are ready to continue. 3. Create the system users and groups that the Nix daemon uses to run @@ -614,14 +614,14 @@ manager. This will happen in a few stages: EOF - if ui_confirm "Would you like to see a more detailed list of what we will do?"; then + if ui_confirm "Would you like to see a more detailed list of what I will do?"; then cat < Date: Thu, 4 Nov 2021 11:41:29 +0100 Subject: [PATCH 14/85] docs: Correct fallback user config path This is in line with XDG Base Directory Specification, where ~/.config is supposed to be used when XDG_CONFIG_HOME is unset. It also better matches the reality, where ~/.config/nix.conf does not seem to be used. --- doc/manual/src/command-ref/conf-file-prefix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 3140170ab..d660db502 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -17,7 +17,7 @@ By default Nix reads settings from the following places: Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and `XDG_CONFIG_HOME`. If these are unset, it will look in - `$HOME/.config/nix.conf`. + `$HOME/.config/nix/nix.conf`. - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. From c4bd6a15c2171153021691776cc657872c42338c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Nov 2021 14:52:35 +0100 Subject: [PATCH 15/85] Add helper function to check whether a function arg is 'X' or '_X' Also allow '_'. --- src/nix/flake.cc | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5855348ac..5eeb5498a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -252,6 +252,14 @@ struct CmdFlakeInfo : CmdFlakeMetadata } }; +static bool argHasName(std::string_view arg, std::string_view expected) +{ + return + arg == expected + || arg == "_" + || (hasPrefix(arg, "_") && arg.substr(1) == expected); +} + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -346,14 +354,14 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->hasFormals() || - (std::string(v.lambda.fun->arg) != "final" && - std::string(v.lambda.fun->arg) != "_final")) + if (!v.isLambda() + || v.lambda.fun->hasFormals() + || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); - if (!body || body->hasFormals() || - (std::string(body->arg) != "prev" && - std::string(body->arg) != "_prev")) + if (!body + || body->hasFormals() + || !argHasName(body->arg, "prev")) throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. From ab35cbd675610a52513f746051e98d0a50815bc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Feb 2020 18:31:16 +0100 Subject: [PATCH 16/85] StaticEnv: Use std::vector instead of std::map --- src/libexpr/eval.cc | 6 +++--- src/libexpr/nixexpr.cc | 25 +++++++++++++++++-------- src/libexpr/nixexpr.hh | 25 +++++++++++++++++++++---- src/libexpr/parser.y | 8 ++++---- src/libexpr/primops.cc | 10 ++++++++-- src/nix/repl.cc | 3 ++- 6 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index db1e7e56d..062d190b6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -583,7 +583,7 @@ Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v2; string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); @@ -609,7 +609,7 @@ Value * EvalState::addPrimOp(const string & name, Value * v = allocValue(); v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); return v; @@ -635,7 +635,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value * v = allocValue(); v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv.vars[envName] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); return v; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 0d0f3e469..95a353a40 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -269,7 +269,7 @@ void ExprVar::bindVars(const StaticEnv & env) if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + auto i = curEnv->find(name); if (i != curEnv->vars.end()) { fromWith = false; this->level = level; @@ -311,14 +311,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv & env) { const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0); if (recursive) { dynamicEnv = &newEnv; unsigned int displ = 0; for (auto & i : attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -342,15 +344,20 @@ void ExprList::bindVars(const StaticEnv & env) void ExprLambda::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv( + false, &env, + (hasFormals() ? formals->formals.size() : 0) + + (arg.empty() ? 0 : 1)); unsigned int displ = 0; - if (!arg.empty()) newEnv.vars[arg] = displ++; + if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); if (hasFormals()) { for (auto & i : formals->formals) - newEnv.vars[i.name] = displ++; + newEnv.vars.emplace_back(i.name, displ++); + + newEnv.sort(); for (auto & i : formals->formals) if (i.def) i.def->bindVars(newEnv); @@ -361,11 +368,13 @@ void ExprLambda::bindVars(const StaticEnv & env) void ExprLet::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, attrs->attrs.size()); unsigned int displ = 0; for (auto & i : attrs->attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 851e875bd..3bfa5e0d3 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -4,8 +4,6 @@ #include "symbol-table.hh" #include "error.hh" -#include - namespace nix { @@ -342,9 +340,28 @@ struct StaticEnv { bool isWith; const StaticEnv * up; - typedef std::map Vars; + + // Note: these must be in sorted order. + typedef std::vector> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; + + StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + vars.reserve(expectedSize); + }; + + void sort() + { + std::sort(vars.begin(), vars.end(), + [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); + } + + Vars::const_iterator find(const Symbol & name) const + { + Vars::value_type key(name, 0); + auto i = std::lower_bound(vars.begin(), vars.end(), key); + if (i != vars.end() && i->first == name) return i; + return vars.end(); + } }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 813ff2fc3..5d0f05206 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -126,14 +126,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs[ad.first] = ad.second; + jAttrs->attrs.emplace(ad.first, ad.second); } } else { dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); e->setName(i->symbol); } } else { @@ -483,7 +483,7 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } } | binds INHERIT '(' expr ')' attrs ';' @@ -492,7 +492,7 @@ binds for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6b3cafec8..8a573f35c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -184,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - StaticEnv staticEnv(false, &state.staticBaseEnv); + StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv.vars[attr.name] = displ; + staticEnv.vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + printTalkative("evaluating file '%1%'", realPath); Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); @@ -3726,6 +3729,7 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ + staticBaseEnv.sort(); sDerivationNix = symbols.create("//builtin/derivation.nix"); eval(parse( #include "primops/derivation.nix.gen.hh" @@ -3735,6 +3739,8 @@ void EvalState::createBaseEnv() /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); + + staticBaseEnv.sort(); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9c0d22438..07bf32a0a 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -644,7 +644,8 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v) { if (displ >= envSize) throw Error("environment full; cannot add more variables"); - staticEnv.vars[name] = displ; + staticEnv.vars.emplace_back(name, displ); + staticEnv.sort(); env->values[displ++] = &v; varNames.insert((string) name); } From 81e7c40264520b387358917987d101f5f5ae4705 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Feb 2020 01:32:01 +0100 Subject: [PATCH 17/85] Optimize primop calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now parse function applications as a vector of arguments rather than as a chain of binary applications, e.g. 'substring 1 2 "foo"' is parsed as ExprCall { .fun = , .args = [ <1>, <2>, <"foo"> ] } rather than ExprApp (ExprApp (ExprApp <1>) <2>) <"foo"> This allows primops to be called immediately (if enough arguments are supplied) without having to allocate intermediate tPrimOpApp values. On $ nix-instantiate --dry-run '' -A nixos.tests.simple.x86_64-linux this gives a substantial performance improvement: user CPU time: median = 0.9209 mean = 0.9218 stddev = 0.0073 min = 0.9086 max = 0.9340 [rejected, p=0.00000, Δ=-0.21433±0.00677] elapsed time: median = 1.0585 mean = 1.0584 stddev = 0.0024 min = 1.0523 max = 1.0623 [rejected, p=0.00000, Δ=-0.20594±0.00236] because it reduces the number of tPrimOpApp allocations from 551990 to 42534 (i.e. only small minority of primop calls are partially applied) which in turn reduces time spent in the garbage collector. --- src/libexpr/eval.cc | 272 +++++++++++++++++++++++------------------ src/libexpr/eval.hh | 12 +- src/libexpr/lexer.l | 1 + src/libexpr/nixexpr.cc | 18 ++- src/libexpr/nixexpr.hh | 12 +- src/libexpr/parser.y | 51 +++++--- 6 files changed, 228 insertions(+), 138 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 062d190b6..b4b4f7b5c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -594,6 +594,8 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { + assert(arity <= maxPrimOpArity); + auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); @@ -1251,144 +1253,182 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } -void ExprApp::eval(EvalState & state, Env & env, Value & v) -{ - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} - - -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->isPrimOpApp()) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.mkPrimOpApp(fun2, &arg); - } -} - -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) { auto trace = evalSettings.traceFunctionCalls ? std::make_unique(pos) : nullptr; forceValue(fun, pos); - if (fun.isPrimOp() || fun.isPrimOpApp()) { - callPrimOp(fun, arg, v, pos); - return; - } + Value vCur(fun); - if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); - } - } + auto makeAppChain = [&]() + { + vRes = vCur; + for (size_t i = 0; i < nrArgs; ++i) { + auto fun2 = allocValue(); + *fun2 = vRes; + vRes.mkPrimOpApp(fun2, args[i]); + } + }; - if (!fun.isLambda()) - throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); + while (nrArgs > 0) { - ExprLambda & lambda(*fun.lambda.fun); + if (vCur.isLambda()) { - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.hasFormals() ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; + ExprLambda & lambda(*vCur.lambda.fun); - size_t displ = 0; + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = vCur.lambda.env; - if (!lambda.hasFormals()) - env2.values[displ++] = &arg; + size_t displ = 0; - else { - forceAttrs(arg, pos); + if (!lambda.hasFormals()) + env2.values[displ++] = args[0]; - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; + else { + forceAttrs(*args[0], pos); - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name); - env2.values[displ++] = i.def->maybeThunk(*this, env2); + if (!lambda.arg.empty()) + env2.values[displ++] = args[0]; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + auto j = args[0]->attrs->get(i.name); + if (!j) { + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *args[0]->attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. */ + try { + lambda.body->eval(*this, env2, vCur); + } catch (Error & e) { + if (loggerSettings.showTrace.get()) { + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); + addErrorTrace(e, pos, "from call site%s", ""); + } + throw; + } + + nrArgs--; + args += 1; + } + + else if (vCur.isPrimOp()) { + + size_t argsLeft = vCur.primOp->arity; + + if (nrArgs < argsLeft) { + /* We don't have enough arguments, so create a tPrimOpApp chain. */ + makeAppChain(); + return; } else { - attrsUsed++; - env2.values[displ++] = j->value; + /* We have all the arguments, so call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[vCur.primOp->name]++; + vCur.primOp->fun(*this, pos, args, vCur); + + nrArgs -= argsLeft; + args += argsLeft; } } - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); - abort(); // can't happen + else if (vCur.isPrimOpApp()) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &vCur; + while (primOp->isPrimOpApp()) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->isPrimOp()); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (nrArgs < argsLeft) { + /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ + makeAppChain(); + return; + } else { + /* We have all the arguments, so call the primop with + the previous and new arguments. */ + + Value * vArgs[arity]; + auto n = argsDone; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) + vArgs[--n] = arg->primOpApp.right; + + for (size_t i = 0; i < argsLeft; ++i) + vArgs[argsDone + i] = args[i]; + + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, vCur); + + nrArgs -= argsLeft; + args += argsLeft; + } } + + else if (vCur.type() == nAttrs) { + if (auto functor = vCur.attrs->get(sFunctor)) { + /* 'vCur" may be allocated on the stack of the calling + function, but for functors we may keep a reference, + so heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue()}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 1, args2, vCur, pos); + } + } + + else + throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); } - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); + vRes = vCur; +} - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (loggerSettings.showTrace.get()) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + +void ExprCall::eval(EvalState & state, Env & env, Value & v) +{ + Value vFun; + fun->eval(state, env, vFun); + + Value * vArgs[args.size()]; + for (size_t i = 0; i < args.size(); ++i) + vArgs[i] = args[i]->maybeThunk(state, env); + + state.callFunction(vFun, args.size(), vArgs, v, pos); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 69119599a..f87dcdd8e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -277,6 +277,8 @@ private: Value * addConstant(const string & name, Value & v); + constexpr static size_t maxPrimOpArity = 3; + Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); @@ -316,8 +318,14 @@ public: bool isFunctor(Value & fun); - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + // FIXME: use std::span + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + + void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + { + Value * args[] = {&arg}; + callFunction(fun, 1, args, vRes, pos); + } /* Automatically call a function for which each argument has a default value or has a binding in the `args' map. */ diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 51593eccd..c18877e29 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -64,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } +// FIXME: optimize static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 95a353a40..372e323bc 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -143,6 +143,16 @@ void ExprLambda::show(std::ostream & str) const str << ": " << *body << ")"; } +void ExprCall::show(std::ostream & str) const +{ + str << '(' << *fun; + for (auto e : args) { + str << ' '; + str << *e; + } + str << ')'; +} + void ExprLet::show(std::ostream & str) const { str << "(let "; @@ -366,6 +376,13 @@ void ExprLambda::bindVars(const StaticEnv & env) body->bindVars(newEnv); } +void ExprCall::bindVars(const StaticEnv & env) +{ + fun->bindVars(env); + for (auto e : args) + e->bindVars(env); +} + void ExprLet::bindVars(const StaticEnv & env) { StaticEnv newEnv(false, &env, attrs->attrs.size()); @@ -461,5 +478,4 @@ size_t SymbolTable::totalSize() const return n; } - } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 3bfa5e0d3..fe12890e9 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -248,6 +248,17 @@ struct ExprLambda : Expr COMMON_METHODS }; +struct ExprCall : Expr +{ + Expr * fun; + std::vector args; + Pos pos; + ExprCall(const Pos & pos, Expr * fun, std::vector && args) + : fun(fun), args(args), pos(pos) + { } + COMMON_METHODS +}; + struct ExprLet : Expr { ExprAttrs * attrs; @@ -306,7 +317,6 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5d0f05206..2e8a04143 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -41,6 +41,12 @@ namespace nix { { }; }; + // Helper to prevent an expensive dynamic_cast call in expr_app. + struct App + { + Expr * e; + bool isCall; + }; } #define YY_DECL int yylex \ @@ -280,10 +286,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err char * uri; std::vector * attrNames; std::vector * string_parts; + nix::App app; // bool == whether this is an ExprCall } %type start expr expr_function expr_if expr_op -%type expr_app expr_select expr_simple +%type expr_select expr_simple +%type expr_app %type expr_list %type binds %type formals @@ -353,13 +361,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } @@ -367,17 +375,24 @@ expr_op | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op { $$ = new ExprConcatStrings(CUR_POS, false, new vector({$1, $3})); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } - | expr_app + | expr_app { $$ = $1.e; } ; expr_app - : expr_app expr_select - { $$ = new ExprApp(CUR_POS, $1, $2); } - | expr_select { $$ = $1; } + : expr_app expr_select { + if ($1.isCall) { + ((ExprCall *) $1.e)->args.push_back($2); + $$ = $1; + } else { + $$.e = new ExprCall(CUR_POS, $1.e, {$2}); + $$.isCall = true; + } + } + | expr_select { $$.e = $1; $$.isCall = false; } ; expr_select @@ -388,7 +403,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } | expr_simple { $$ = $1; } ; @@ -412,10 +427,10 @@ expr_simple } | SPATH { string path($1 + 1, strlen($1) - 2); - $$ = new ExprApp(CUR_POS, - new ExprApp(new ExprVar(data->symbols.create("__findFile")), - new ExprVar(data->symbols.create("__nixPath"))), - new ExprString(data->symbols.create(path))); + $$ = new ExprCall(CUR_POS, + new ExprVar(data->symbols.create("__findFile")), + {new ExprVar(data->symbols.create("__nixPath")), + new ExprString(data->symbols.create(path))}); } | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); From bcf47800067243b124569da9019077c81bf71cd5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Feb 2020 14:33:01 +0100 Subject: [PATCH 18/85] Add level / displacement types --- src/libexpr/eval.cc | 8 ++++---- src/libexpr/nixexpr.cc | 10 +++++----- src/libexpr/nixexpr.hh | 11 +++++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b4b4f7b5c..641687f2a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -787,7 +787,7 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (size_t l = var.level; l; --l, env = env->up) ; + for (auto l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; @@ -1060,7 +1060,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -1136,7 +1136,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -1283,7 +1283,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & Env & env2(allocEnv(size)); env2.up = vCur.lambda.env; - size_t displ = 0; + Displacement displ = 0; if (!lambda.hasFormals()) env2.values[displ++] = args[0]; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 372e323bc..57c2f6e44 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -273,7 +273,7 @@ void ExprVar::bindVars(const StaticEnv & env) /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; - unsigned int level; + Level level; int withLevel = -1; for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { @@ -326,7 +326,7 @@ void ExprAttrs::bindVars(const StaticEnv & env) if (recursive) { dynamicEnv = &newEnv; - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs) newEnv.vars.emplace_back(i.first, i.second.displ = displ++); @@ -359,7 +359,7 @@ void ExprLambda::bindVars(const StaticEnv & env) (hasFormals() ? formals->formals.size() : 0) + (arg.empty() ? 0 : 1)); - unsigned int displ = 0; + Displacement displ = 0; if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); @@ -387,7 +387,7 @@ void ExprLet::bindVars(const StaticEnv & env) { StaticEnv newEnv(false, &env, attrs->attrs.size()); - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) newEnv.vars.emplace_back(i.first, i.second.displ = displ++); @@ -405,7 +405,7 @@ void ExprWith::bindVars(const StaticEnv & env) level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; - unsigned int level; + Level level; prevWith = 0; for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index fe12890e9..13256272c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -133,6 +133,9 @@ struct ExprPath : Expr Value * maybeThunk(EvalState & state, Env & env); }; +typedef uint32_t Level; +typedef uint32_t Displacement; + struct ExprVar : Expr { Pos pos; @@ -148,8 +151,8 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name' from the set stored in the environment that is `level' levels up from the current one.*/ - unsigned int level; - unsigned int displ; + Level level; + Displacement displ; ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; @@ -183,7 +186,7 @@ struct ExprAttrs : Expr bool inherited; Expr * e; Pos pos; - unsigned int displ; // displacement + Displacement displ; // displacement AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; @@ -352,7 +355,7 @@ struct StaticEnv const StaticEnv * up; // Note: these must be in sorted order. - typedef std::vector> Vars; + typedef std::vector> Vars; Vars vars; StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { From cbfbf71e0821fd79ae4837e7fe03c051437f43dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Mar 2020 17:22:31 +0100 Subject: [PATCH 19/85] Use callFunction() with an array for some calls with arity > 1 --- src/libexpr/primops.cc | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a573f35c..9b166d63c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1883,9 +1883,6 @@ static void addPath( Value arg1; mkString(arg1, path); - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); - Value arg2; mkString(arg2, S_ISREG(st.st_mode) ? "regular" : @@ -1893,8 +1890,9 @@ static void addPath( S_ISLNK(st.st_mode) ? "symlink" : "unknown" /* not supported, will fail! */); + Value * args []{&arg1, &arg2}; Value res; - state.callFunction(fun2, arg2, res, noPos); + state.callFunction(*filterFun, 2, args, res, pos); return state.forceBool(res, pos); }) : defaultPathFilter; @@ -2695,10 +2693,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value * vCur = args[1]; for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); + Value * vs []{vCur, args[2]->listElems()[n]}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + state.callFunction(*args[0], 2, vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -2819,17 +2816,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[1]->listElems()[n]; } - auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) return CompareValues()(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); + Value * vs[] = {a, b}; + Value vBool; + state.callFunction(*args[0], 2, vs, vBool, pos); + return state.forceBool(vBool, pos); }; /* FIXME: std::sort can segfault if the comparator is not a strict From acd6bddec706e2f180279ca79a0c1a4abac416d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Mar 2020 18:15:06 +0100 Subject: [PATCH 20/85] Fix derivation primop --- src/libexpr/eval.cc | 14 ++++++++++---- src/libexpr/eval.hh | 2 ++ src/libexpr/primops.cc | 13 ++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 641687f2a..d371a3fb8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -583,14 +583,20 @@ Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addConstant(name, v2); return v2; } +void EvalState::addConstant(const string & name, Value * v) +{ + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); +} + + Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f87dcdd8e..65cf04b57 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -277,6 +277,8 @@ private: Value * addConstant(const string & name, Value & v); + void addConstant(const string & name, Value * v); + constexpr static size_t maxPrimOpArity = 3; Value * addPrimOp(const string & name, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9b166d63c..e4107dbe1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3725,18 +3725,21 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - staticBaseEnv.sort(); sDerivationNix = symbols.create("//builtin/derivation.nix"); - eval(parse( - #include "primops/derivation.nix.gen.hh" - , foFile, sDerivationNix, "/", staticBaseEnv), v); - addConstant("derivation", v); + auto vDerivation = allocValue(); + addConstant("derivation", vDerivation); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); staticBaseEnv.sort(); + + /* Note: we have to initialize the 'derivation' constant *after* + building baseEnv/staticBaseEnv because it uses 'builtins'. */ + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); } From 05560f6350f5e330c286e048b8fbbb24dafbf16b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Mar 2020 18:16:49 +0100 Subject: [PATCH 21/85] Fix function-trace test case --- tests/function-trace.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/function-trace.sh b/tests/function-trace.sh index 3b7f364e3..0b7f49d82 100755 --- a/tests/function-trace.sh +++ b/tests/function-trace.sh @@ -60,8 +60,6 @@ function-trace exited (string):1:1 at expect_trace '(x: x) 1 2' " function-trace entered (string):1:1 at function-trace exited (string):1:1 at -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at " # Not a function From 40925337a972991468335b90ce4f685f7102d830 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Mar 2020 22:41:56 +0100 Subject: [PATCH 22/85] Remove maxPrimOpArity --- src/libexpr/eval.cc | 2 -- src/libexpr/eval.hh | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d371a3fb8..402de78ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -600,8 +600,6 @@ void EvalState::addConstant(const string & name, Value * v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { - assert(arity <= maxPrimOpArity); - auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 65cf04b57..1aab8e166 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -279,8 +279,6 @@ private: void addConstant(const string & name, Value * v); - constexpr static size_t maxPrimOpArity = 3; - Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); From abdf9f2a6ecdc364531ed1554b689a6a467b79d1 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 4 Nov 2021 14:09:40 -0500 Subject: [PATCH 23/85] darwin-install: fix already-mounted store volumes This adds an explicit unmount of the store volume to avoid cases where the installer can hang in await_volume when: - the user already has a store volume - that volume is already mounted somewhere other than /nix - they do not take a path through the installer that results in an explicit unmount (as both removing and encrypting the volume would do) --- scripts/create-darwin-volume.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index b52232dd3..334b75045 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -742,6 +742,9 @@ setup_volume() { use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}" + _sudo "to ensure the Nix volume is not mounted" \ + /usr/sbin/diskutil unmount force "$use_special" || true # might not be mounted + use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")} setup_fstab "$use_uuid" From 1a4c9ba50bb36ab5d18f47b0d00052d274ed824f Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 3 Nov 2021 10:54:17 +0100 Subject: [PATCH 24/85] =?UTF-8?q?Fix=20`nix=20repl`=E2=80=99s=20building?= =?UTF-8?q?=20of=20CA=20derivations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When running a `:b` command in the repl, after building the derivations query the store for its outputs rather than just assuming that they are known in the derivation itself (which isn’t true for CA derivations) Fix #5328 --- src/nix/repl.cc | 4 ++-- tests/ca/repl.sh | 5 +++++ tests/local.mk | 2 +- tests/repl.sh | 4 +++- 4 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 tests/ca/repl.sh diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9c0d22438..bc31cf1d4 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -504,8 +504,8 @@ bool NixRepl::processLine(string line) state->store->buildPaths({DerivedPath::Built{drvPath}}); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & i : drv.outputsAndOptPaths(*state->store)) - logger->cout(" %s -> %s", i.first, state->store->printStorePath(*i.second.second)); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); } else { diff --git a/tests/ca/repl.sh b/tests/ca/repl.sh new file mode 100644 index 000000000..3808c7cb2 --- /dev/null +++ b/tests/ca/repl.sh @@ -0,0 +1,5 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source repl.sh diff --git a/tests/local.mk b/tests/local.mk index 8d454ee19..b63a12d96 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -48,7 +48,7 @@ nix_tests = \ flakes.sh \ build.sh \ compute-levels.sh \ - repl.sh \ + repl.sh ca/repl.sh \ ca/build.sh \ ca/build-with-garbage-path.sh \ ca/duplicate-realisation-in-closure.sh \ diff --git a/tests/repl.sh b/tests/repl.sh index 4e3059517..d360821f2 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -7,7 +7,9 @@ simple = import ./simple.nix testRepl () { local nixArgs=("$@") - local outPath=$(nix repl "${nixArgs[@]}" <<< "$replCmds" |& + local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" + echo "$replOutput" + local outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" } From 93eadd5803caa6d28be056c58194be974a87aeb8 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Nov 2021 11:11:33 +0100 Subject: [PATCH 25/85] Make the post-build-hook use the daemon Nix package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having the `post-build-hook` use `nix` from the client package can lead to a deadlock in case there’s a db migration to do between both, as a `nix` command running inside the hook will run as root (and as such will bypass the daemon), so might trigger a db migration, which will get stuck trying to get a global lock on the DB (as the daemon that ran the hook already has a lock on it). --- tests/common.sh.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 08f5e0a77..3d0d56120 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -36,8 +36,9 @@ export PATH=@bindir@:$PATH if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH fi +DAEMON_PATH="$PATH" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon" + DAEMON_PATH="${NIX_DAEMON_PACKAGE}:$DAEMON_PATH" fi coreutils=@coreutils@ @@ -89,7 +90,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_DAEMON_SOCKET_PATH - ${NIX_DAEMON_COMMAND:-nix daemon} & + PATH=$DAEMON_PATH nix daemon & for ((i = 0; i < 30; i++)); do if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi sleep 1 From 1f3c3a3785b21fe349480b562d5f97b8cc9caa6d Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Nov 2021 16:17:49 +0100 Subject: [PATCH 26/85] Make the flake options work when using the daemon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When setting flake-local options (with the `nixConfig` field), forward these options to the daemon in case we’re using one. This is necessary in particular for options like `binary-caches` or `post-build-hook` to make sense. Fix --- src/libexpr/flake/flake.cc | 2 +- src/libstore/remote-store.cc | 4 ++++ src/libstore/remote-store.hh | 2 ++ src/libstore/store-api.hh | 5 +++++ tests/flake-local-settings.sh | 35 +++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 6 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/flake-local-settings.sh diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c9d848495..07ed3caa2 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -307,7 +307,7 @@ LockedFlake lockFlake( if (lockFlags.applyNixConfig) { flake.config.apply(); - // FIXME: send new config to the daemon. + state.store->setOptions(); } try { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7decc059c..274203f8d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -290,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection() return ConnectionHandle(connections->get()); } +void RemoteStore::setOptions() +{ + setOptions(*(getConnection().handle)); +} bool RemoteStore::isValidPathUncached(const StorePath & path) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a3036e6b0..5f6da9af5 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -147,6 +147,8 @@ protected: virtual void setOptions(Connection & conn); + void setOptions() override; + ConnectionHandle getConnection(); friend struct ConnectionHandle; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7d02340df..8472e726a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -724,6 +724,11 @@ public: virtual void createUser(const std::string & userName, uid_t userId) { } + /* + * Synchronises the options of the client with those of the daemon + * (a no-op when there’s no daemon) + */ + virtual void setOptions() { } protected: Stats stats; diff --git a/tests/flake-local-settings.sh b/tests/flake-local-settings.sh new file mode 100644 index 000000000..c037431c8 --- /dev/null +++ b/tests/flake-local-settings.sh @@ -0,0 +1,35 @@ +source common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local + +cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME + +cd $TEST_HOME + +rm -f post-hook-ran +cat < echoing-post-hook.sh +#!/bin/sh + +echo "ThePostHookRan" > $PWD/post-hook-ran +EOF +chmod +x echoing-post-hook.sh + +cat < flake.nix +{ + nixConfig.post-build-hook = "$PWD/echoing-post-hook.sh"; + + outputs = a: { + defaultPackage.$system = import ./simple.nix; + }; +} +EOF + +# Ugly hack for testing +mkdir -p .local/share/nix +cat < .local/share/nix/trusted-settings.json +{"post-build-hook":{"$PWD/echoing-post-hook.sh":true}} +EOF + +nix build +test -f post-hook-ran || fail "The post hook should have ran" diff --git a/tests/local.mk b/tests/local.mk index 8d454ee19..4c53f09f2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -46,6 +46,7 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ + flake-local-settings.sh \ build.sh \ compute-levels.sh \ repl.sh \ From 9d4dcff37a14c5bab98761e417a2335a9ac68c96 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Sat, 23 Oct 2021 21:31:46 +0300 Subject: [PATCH 27/85] addPath: allow paths with references Since 4806f2f6b0fd2cae401b89fe19d8c528ffd88b5f, we can't have paths with references passed to builtins.{path,filterSource}. This prevents many cases of those functions called on IFD outputs from working. Resolve this by passing the references found in the original path to the added path. --- src/libexpr/primops.cc | 10 +++++----- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/daemon.cc | 2 -- src/libstore/local-store.cc | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4e0eda7f3..4a615c59d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1859,12 +1859,12 @@ static void addPath( // be rewritten to the actual output). state.realiseContext(context); + StorePathSet refs = StorePathSet(); + if (state.store->isInStore(path)) { auto [storePath, subPath] = state.store->toStorePath(path); - auto info = state.store->queryPathInfo(storePath); - if (!info->references.empty()) - throw EvalError("store path '%s' is not allowed to have references", - state.store->printStorePath(storePath)); + // FIXME: we should scanForReferences on the path before adding it + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } @@ -1904,7 +1904,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 280f1d4b5..a039c57b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -314,7 +314,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + makeFixedOutputPath(method, nar.first, name, references), nar.first, }; info.narSize = nar.second; @@ -405,7 +405,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + makeFixedOutputPath(method, h, name, references), nar.first, }; info.narSize = nar.second; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fa6606ba4..b1cc07486 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -402,8 +402,6 @@ static void performOp(TunnelLogger * logger, ref store, return store->queryPathInfo(path); }, [&](FixedOutputHashMethod & fohm) { - // if (!refs.empty()) - // throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 85f1de0b0..33ac77af5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1358,7 +1358,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto dstPath = makeFixedOutputPath(method, hash, name, references); addTempRoot(dstPath); From 8e7359db6496ee72a84322d46afa322604afd1a9 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sun, 7 Nov 2021 18:21:24 +0100 Subject: [PATCH 28/85] Remove unused "" symbol The requirement for the symbol has been removed since at least 7d47498. --- src/libexpr/parser.y | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 2e8a04143..923997bf6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -33,11 +33,9 @@ namespace nix { Symbol file; FileOrigin origin; std::optional error; - Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , sLetBody(symbols.create("")) { }; }; From d589782fb0b8e73f562792ba9d02339e32cb2d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Zimmermann?= Date: Mon, 8 Nov 2021 08:24:15 +0100 Subject: [PATCH 29/85] Fix some typos in CLI guideline. --- doc/manual/src/contributing/cli-guideline.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 69bc45691..258f090fb 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -240,7 +240,7 @@ love, but if not done perfectly it will annoy users and leave bad impression. Input to a command is provided via `ARGUMENTS` and `OPTIONS`. `ARGUMENTS` represent a required input for a function. When choosing to use -`ARGUMENT` over function please be aware of the downsides that come with it: +`ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it: - User will need to remember the order of `ARGUMENTS`. This is not a problem if there is only one `ARGUMENT`. @@ -253,7 +253,7 @@ developer consider the downsides and choose wisely. ## Naming the `OPTIONS` -Then only naming convention - apart from the ones mentioned in Naming the +The only naming convention - apart from the ones mentioned in Naming the `COMMANDS` section is how flags are named. Flags are a type of `OPTION` that represent an option that can be turned ON of @@ -271,7 +271,7 @@ to improve the discoverability of possible input. A new user will most likely not know which `ARGUMENTS` and `OPTIONS` are required or which values are possible for those options. -In cases, the user might not provide the input or they provide wrong input, +In case the user does not provide the input or they provide wrong input, rather than show the error, prompt a user with an option to find and select correct input (see examples). From ff2af4d64ee9789c3c50f7e49897e8fa9fda6e16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Nov 2021 22:00:45 +0100 Subject: [PATCH 30/85] Unshare mount namespace in main() Doing it as a side-effect of calling LocalStore::makeStoreWritable() is very ugly. Also, make sure that stopping the progress bar joins the update thread, otherwise that thread should be unshared as well. --- src/libmain/progress-bar.cc | 16 +++++++++------- src/libstore/local-store.cc | 4 ---- src/nix/main.cc | 8 ++++++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index b2a6e2a82..63955eed1 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -103,17 +103,19 @@ public: ~ProgressBar() { stop(); - updateThread.join(); } void stop() override { - auto state(state_.lock()); - if (!state->active) return; - state->active = false; - writeToStderr("\r\e[K"); - updateCV.notify_one(); - quitCV.notify_one(); + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + writeToStderr("\r\e[K"); + updateCV.notify_one(); + quitCV.notify_one(); + } + updateThread.join(); } bool isVerbose() override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1cef50a40..eb3457339 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -504,10 +504,6 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 1e033f4f2..01889a71f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -255,6 +255,14 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + #if __linux__ + if (getuid() == 0) { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } + #endif + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); From 732dd904280c60f64744aed9df51908ce000016b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 22:00:58 +0000 Subject: [PATCH 31/85] Bump actions/checkout from 2.3.5 to 2.4.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.5 to 2.4.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.5...v2.4.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cdf6ddb67..492ee5388 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - uses: cachix/install-nix-action@v14.1 @@ -46,7 +46,7 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV @@ -67,7 +67,7 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v14.1 with: From 0b005bc9d67b3f621bc78e5ecb2cd834172d5563 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 9 Nov 2021 12:24:49 +0300 Subject: [PATCH 32/85] addToStore, addToStoreFromDump: refactor: pass refs by const reference Co-Authored-By: Eelco Dolstra --- src/libexpr/primops.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/binary-cache-store.hh | 4 ++-- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 4 ++-- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4a615c59d..047856abc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1859,7 +1859,7 @@ static void addPath( // be rewritten to the actual output). state.realiseContext(context); - StorePathSet refs = StorePathSet(); + StorePathSet refs; if (state.store->isInStore(path)) { auto [storePath, subPath] = state.store->toStorePath(path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a039c57b3..065cac8b5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -308,7 +308,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); @@ -386,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, StorePathSet references) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index d8f077db2..a703994d3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -97,11 +97,11 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references ) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair, StorePathSet references) override; + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 694c408f3..383c6b990 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1180,7 +1180,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, - StorePathSet references = StorePathSet()) override + const StorePathSet & references = StorePathSet()) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1200,7 +1200,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - StorePathSet references = StorePathSet()) override + const StorePathSet & references = StorePathSet()) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 433ddc8e1..c4fb91364 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair, StorePathSet references) override + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 33ac77af5..444308d88 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1302,7 +1302,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7a7329dd8..c6441ef01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -140,7 +140,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, StorePathSet references) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 632dfcf21..0db442579 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -578,7 +578,7 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, StorePathSet references) + FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 31eb1871a..9c3e0bf05 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -73,7 +73,7 @@ public: /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, StorePathSet references = StorePathSet()) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 84d6db297..329cf679f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -237,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, StorePathSet references) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c4fdb176a..34d4dac3c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,7 +453,7 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, StorePathSet references = StorePathSet()); + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -470,7 +470,7 @@ public: // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - StorePathSet references = StorePathSet()) + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is From f7859eef49ea3a3563d821ba17c29deb7757b8a4 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 8 Nov 2021 20:08:09 -0600 Subject: [PATCH 33/85] installer: improve existing rc backup nag --- scripts/install-multi-user.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7d1cb8c5a..eb5c48b73 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -387,19 +387,27 @@ EOF fi for profile_target in "${PROFILE_TARGETS[@]}"; do + # TODO: I think it would be good to accumulate a list of all + # of the copies so that people don't hit this 2 or 3x in + # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + # this backup process first released in Nix 2.1 failure < Date: Tue, 9 Nov 2021 11:14:15 -0600 Subject: [PATCH 34/85] Recognize singular "nixosModule" in nix flake show This makes nixosModule appears as a "NixOS Module" like nixosModules does. --- src/nix/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5eeb5498a..97f4d911c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1052,7 +1052,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 1 && attrPath[0] == "overlay") || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : - attrPath.size() == 2 && attrPath[0] == "nixosModules" ? std::make_pair("nixos-module", "NixOS module") : + (attrPath.size() == 1 && attrPath[0] == "nixosModule") + || (attrPath.size() == 2 && attrPath[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : std::make_pair("unknown", "unknown"); if (json) { j.emplace("type", type); From cc78901ccbfd1e7b12da7b24ea0866e53db3dac8 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 10 Nov 2021 14:06:18 -0600 Subject: [PATCH 35/85] installer: make rc replacement instructions explicit --- scripts/install-multi-user.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index eb5c48b73..eba894799 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -406,8 +406,9 @@ Here's how to clean up the old backup file: Nix-related in it. If it does, something is probably quite wrong. Please open an issue or get in touch immediately. -3. Once you confirm $profile_target is backed up and doesn't already - mention Nix, replace it with $profile_target$PROFILE_BACKUP_SUFFIX. +3. Once you confirm $profile_target is backed up and + $profile_target$PROFILE_BACKUP_SUFFIX doesn't mention Nix, run: + mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target EOF fi done From 07bffe799853ef5e2757b9892e4e3dac89bfccbb Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Wed, 10 Nov 2021 14:56:22 +0300 Subject: [PATCH 36/85] Flakes: refetch the input when a follows disappears When an input follows disappears, we can't just reuse the old lock file entries since we may be missing some required ones. Refetch the input when this happens. Closes https://github.com/NixOS/nix/issues/5289 --- src/libexpr/flake/flake.cc | 14 ++++++++++++++ tests/flakes.sh | 28 +++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 07ed3caa2..1dc6f5694 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -462,6 +462,8 @@ LockedFlake lockFlake( those. */ FlakeInputs fakeInputs; + bool refetch = false; + for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -469,12 +471,24 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { + auto o = input.overrides.find(i.first); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == input.overrides.end()) { + refetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, }); } } + if (refetch) + fakeInputs = getFlake(state, oldLock->lockedRef, false, flakeCache).inputs; + computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 57d1b9aad..9e10322b9 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -707,10 +707,10 @@ cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix < Date: Sun, 31 Oct 2021 00:22:35 +0200 Subject: [PATCH 37/85] Docker image with Nix inside --- docker.nix | 264 +++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 7 ++ 2 files changed, 271 insertions(+) create mode 100644 docker.nix diff --git a/docker.nix b/docker.nix new file mode 100644 index 000000000..40af2f38b --- /dev/null +++ b/docker.nix @@ -0,0 +1,264 @@ +{ pkgs ? import { } +, lib ? pkgs.lib +, name ? "nix" +, tag ? "latest" +, crossSystem ? null +, channelName ? "nixpkgs" +, channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +}: +let + buildPkgs = pkgs; + targetPkgs = + if crossSystem != null && crossSystem != pkgs.system + then { + aarch64-linux = pkgs.pkgsCross.aarch64-multiplatform; + armv7l-linux = pkgs.pkgsCross.armv7l-hf-multiplatform.system; + x86_64-linux = pkgs.pkgsCross.gnu64; + powerpc64le-linux = pkgs.pkgsCross.musl-power; + i686-linux = pkgs.pkgsCross.gnu32; + }.${crossSystem} + else pkgs; + + defaultPkgs = [ + targetPkgs.nix + targetPkgs.bashInteractive + targetPkgs.coreutils-full + targetPkgs.gnutar + targetPkgs.gzip + targetPkgs.gnugrep + targetPkgs.which + targetPkgs.curl + targetPkgs.less + targetPkgs.wget + targetPkgs.man + targetPkgs.cacert.out + targetPkgs.findutils + ]; + + users = { + + root = { + uid = 0; + shell = "/bin/bash"; + home = "/root"; + gid = 0; + }; + + } // lib.listToAttrs ( + map + ( + n: { + name = "nixbld${toString n}"; + value = { + uid = 30000 + n; + gid = 30000; + groups = [ "nixbld" ]; + description = "Nix build user ${toString n}"; + }; + } + ) + (lib.lists.range 1 32) + ); + + groups = { + root.gid = 0; + nixbld.gid = 30000; + }; + + userToPasswd = ( + k: + { uid + , gid ? 65534 + , home ? "/var/empty" + , description ? "" + , shell ? "/bin/false" + , groups ? [ ] + }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}" + ); + passwdContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToPasswd users)) + ); + + userToShadow = k: { ... }: "${k}:!:1::::::"; + shadowContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToShadow users)) + ); + + # Map groups to members + # { + # group = [ "user1" "user2" ]; + # } + groupMemberMap = ( + let + # Create a flat list of user/group mappings + mappings = ( + builtins.foldl' + ( + acc: user: + let + groups = users.${user}.groups or [ ]; + in + acc ++ map + (group: { + inherit user group; + }) + groups + ) + [ ] + (lib.attrNames users) + ); + in + ( + builtins.foldl' + ( + acc: v: acc // { + ${v.group} = acc.${v.group} or [ ] ++ [ v.user ]; + } + ) + { } + mappings) + ); + + groupToGroup = k: { gid }: + let + members = groupMemberMap.${k} or [ ]; + in + "${k}:x:${toString gid}:${lib.concatStringsSep "," members}"; + groupContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs groupToGroup groups)) + ); + + nixConf = { + sandbox = "false"; + build-users-group = "nixbld"; + trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + }; + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + baseSystem = + let + nixpkgs = targetPkgs.path; + channel = targetPkgs.runCommand "channel-nixos" { } '' + mkdir $out + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''; + rootEnv = pkgs.buildEnv { + name = "root-profile-env"; + paths = defaultPkgs; + }; + profile = targetPkgs.runCommand "user-environment" { } '' + mkdir $out + cp -a ${rootEnv}/* $out/ + + cat > $out/manifest.nix < $out/etc/passwd + echo "" >> $out/etc/passwd + + cat $groupContentsPath > $out/etc/group + echo "" >> $out/etc/group + + cat $shadowContentsPath > $out/etc/shadow + echo "" >> $out/etc/shadow + + mkdir -p $out/usr + ln -s /nix/var/nix/profiles/share $out/usr/ + + mkdir -p $out/nix/var/nix/gcroots + + mkdir $out/tmp + + mkdir -p $out/etc/nix + cat $nixConfContentsPath > $out/etc/nix/nix.conf + + mkdir -p $out/root + mkdir -p $out/nix/var/nix/profiles/per-user/root + + ln -s ${profile} $out/nix/var/nix/profiles/default-1-link + ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default $out/root/.nix-profile + + ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link + ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + + mkdir -p $out/root/.nix-defexpr + ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out/root/.nix-channels + + mkdir -p $out/bin $out/usr/bin + ln -s ${targetPkgs.coreutils}/bin/env $out/usr/bin/env + ln -s ${targetPkgs.bashInteractive}/bin/bash $out/bin/sh + ''; + +in +targetPkgs.dockerTools.buildLayeredImageWithNixDb { + + inherit name tag; + + contents = [ baseSystem ]; + + extraCommands = '' + rm -rf nix-support + ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles + ''; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + +} diff --git a/flake.nix b/flake.nix index ed622ec86..fd5e18429 100644 --- a/flake.nix +++ b/flake.nix @@ -405,6 +405,13 @@ installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + # docker image with Nix inside + dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: + import ./docker.nix { + pkgs = nixpkgsFor.${system}; + tag = version; + }); + # Line coverage analysis. coverage = with nixpkgsFor.x86_64-linux; From c1bf9e39f1138bc43fdfa0c649af33e63f57617a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Nov 2021 14:03:55 +0100 Subject: [PATCH 38/85] docker.nix: Use 'with' Co-authored-by: Sandro --- docker.nix | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docker.nix b/docker.nix index 40af2f38b..316d57a36 100644 --- a/docker.nix +++ b/docker.nix @@ -19,20 +19,20 @@ let }.${crossSystem} else pkgs; - defaultPkgs = [ - targetPkgs.nix - targetPkgs.bashInteractive - targetPkgs.coreutils-full - targetPkgs.gnutar - targetPkgs.gzip - targetPkgs.gnugrep - targetPkgs.which - targetPkgs.curl - targetPkgs.less - targetPkgs.wget - targetPkgs.man - targetPkgs.cacert.out - targetPkgs.findutils + defaultPkgs = with targetPkgs; [ + nix + bashInteractive + coreutils-full + gnutar + gzip + gnugrep + which + curl + less + wget + man + cacert.out + findutils ]; users = { From a118a70649ca2be9c36e526269ee54bc87ea06d4 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Thu, 11 Nov 2021 16:03:09 +0100 Subject: [PATCH 39/85] Documenting how to use/build Nix' Docker image --- doc/manual/src/SUMMARY.md.in | 1 + .../src/installation/installing-docker.md | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 doc/manual/src/installation/installing-docker.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3869f4791..8d9b061ba 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -9,6 +9,7 @@ - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) + - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - [Single-User Mode](installation/single-user.md) - [Multi-User Mode](installation/multi-user.md) diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md new file mode 100644 index 000000000..3d2255b7a --- /dev/null +++ b/doc/manual/src/installation/installing-docker.md @@ -0,0 +1,59 @@ +# Using Nix within Docker + +To run the latest stable release of Nix with Docker run the following command: + +```console +$ docker -ti run nixos/nix +Unable to find image 'nixos/nix:latest' locally +latest: Pulling from nixos/nix +5843afab3874: Pull complete +b52bf13f109c: Pull complete +1e2415612aa3: Pull complete +Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff +Status: Downloaded newer image for nixos/nix:latest +35ca4ada6e96:/# nix --version +nix (Nix) 2.3.12 +35ca4ada6e96:/# exit +``` + +# What is included in Nix' Docker image? + +The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` +(and not with `Dockerfile` as it is usual with Docker images). You can still +base your custom Docker image on it as you would do with any other Docker +image. + +The Docker image is also not based on any other image and includes minimal set +of runtime dependencies that are required to use Nix: + + - pkgs.nix + - pkgs.bashInteractive + - pkgs.coreutils-full + - pkgs.gnutar + - pkgs.gzip + - pkgs.gnugrep + - pkgs.which + - pkgs.curl + - pkgs.less + - pkgs.wget + - pkgs.man + - pkgs.cacert.out + - pkgs.findutils + +# Docker image with the latest development version of Nix + +To get the latest image that was built by [Hydra](https://hydra.nixos.org) run +the following command: + +```console +$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load +$ docker run -ti nix:2.5pre20211105 +``` + +You can also build a Docker image from source yourself: + +```console +$ nix build ./\#hydraJobs.dockerImage.x86_64-linux +$ docker load -i ./result +$ docker run -ti nix:2.5pre20211105 +``` From c1dea92dd6fb5c07127bcf43d8c76d8a59470d7a Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Thu, 11 Nov 2021 21:29:18 -0500 Subject: [PATCH 40/85] nix key: Fix error message and don't require flakes --- src/nix/sigs.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6a238efbe..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -218,8 +218,7 @@ struct CmdKey : NixMultiCommand void run() override { if (!command) - throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); + throw UsageError("'nix key' requires a sub-command."); command->second->prepare(); command->second->run(); } From 30e5c5c55fb84a103575fda72fa11c7b1b88d928 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Fri, 12 Nov 2021 03:18:31 -0500 Subject: [PATCH 41/85] nix registry: Mark experimental This is part of the flakes feature. Mark it as such. --- src/nix/registry.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 6a92576c7..c496f94f8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -226,6 +226,7 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { + settings.requireExperimentalFeature(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); command->second->prepare(); From bceda304982a34c65d5a1dab449cb5bc59f63b83 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Nov 2021 13:41:15 +0100 Subject: [PATCH 42/85] Typo --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2ba79ec46..589b449d0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1993,7 +1993,7 @@ void LocalDerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); From 30496af5980fd03706f587eef014e630e9d9d318 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 12 Nov 2021 09:50:07 -0500 Subject: [PATCH 43/85] Adds an accept-flake-config flag --- src/libexpr/flake/config.cc | 23 +++++++++++------------ src/libstore/globals.hh | 3 +++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 41b6f78ed..c03f4106c 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "globals.hh" #include @@ -52,21 +53,19 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { + if (nix::settings.acceptFlakeConfig){ + trusted = true; + } else if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { trusted = *saved; + warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { + trusted = true; + } + if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') { + trustedList[name][valueS] = trusted; + writeTrustedList(trustedList); } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 165639261..a50eb6803 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -951,6 +951,9 @@ public: Setting useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references."}; + + Setting acceptFlakeConfig{this, false, "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting."}; }; From 83af9550a166ece787bbea6014d827d3e3af277b Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 12 Nov 2021 16:02:32 +0100 Subject: [PATCH 44/85] Add a test for the `--accept-flake-config` option --- tests/flake-local-settings.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/flake-local-settings.sh b/tests/flake-local-settings.sh index c037431c8..6ee6b17d6 100644 --- a/tests/flake-local-settings.sh +++ b/tests/flake-local-settings.sh @@ -25,11 +25,5 @@ cat < flake.nix } EOF -# Ugly hack for testing -mkdir -p .local/share/nix -cat < .local/share/nix/trusted-settings.json -{"post-build-hook":{"$PWD/echoing-post-hook.sh":true}} -EOF - -nix build +nix build --accept-flake-config test -f post-hook-ran || fail "The post hook should have ran" From d0e9e184895b62d193a6eff5e332a338d3664e5b Mon Sep 17 00:00:00 2001 From: Kevin Amado Date: Sat, 13 Nov 2021 20:29:31 -0500 Subject: [PATCH 45/85] toXML: display errors position - This change applies to builtins.toXML and inner workings - Proof of concept: ```nix let e = builtins.toXML e; in e ``` - Before: ``` $ nix-instantiate --eval poc.nix error: infinite recursion encountered ``` - After: ``` $ nix-instantiate --eval poc.nix error: infinite recursion encountered at /data/github/kamadorueda/nix/poc.nix:1:9: 1| let e = builtins.toXML e; in e | ``` --- src/libexpr/primops.cc | 2 +- src/libexpr/value-to-xml.cc | 25 ++++++++++++++----------- src/libexpr/value-to-xml.hh | 4 ++-- src/libexpr/value.hh | 3 ++- src/nix-instantiate/nix-instantiate.cc | 2 +- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aaeafb931..668ad00ec 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1582,7 +1582,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value { std::ostringstream out; PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); + printValueAsXML(state, true, false, *args[0], out, context, pos); mkString(v, out.str(), context); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index b44455f5f..54268ece0 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -46,17 +47,18 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); + *a.value, doc, context, drvsSeen, *a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -121,7 +123,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen, pos); break; } @@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: @@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, PathSet & context, const Pos & pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..cc778a2cb 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - + Value & v, std::ostream & out, PathSet & context, const Pos & pos); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a1f131f9e..3bb97b3c2 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -94,7 +94,8 @@ class ExternalValueBase /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const; virtual ~ExternalValueBase() { diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 25d0fa3ba..6ff3d2f19 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -50,7 +50,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context); + printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) printValueAsJSON(state, strict, vRes, std::cout, context); else { From eae54f2d5208427080c8793d6c88db8550337d77 Mon Sep 17 00:00:00 2001 From: Kevin Amado Date: Sat, 13 Nov 2021 22:28:20 -0500 Subject: [PATCH 46/85] fix many doc typos --- doc/manual/src/contributing/cli-guideline.md | 24 +++++++++---------- doc/manual/src/expressions/builtins-prefix.md | 2 +- .../package-management/garbage-collection.md | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 258f090fb..01a1b1e73 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -103,7 +103,7 @@ impacted the most by bad user experience. # Help is essential Help should be built into your command line so that new users can gradually -discover new features when they need them. +discover new features when they need them. ## Looking for help @@ -176,7 +176,7 @@ $ nix init --template=template#pyton ------------------------------------------------------------------------ Initializing Nix project at `/path/to/here`. Select a template for you new project: - |> template#pyton + |> template#python template#python-pip template#python-poetry ``` @@ -237,7 +237,7 @@ love, but if not done perfectly it will annoy users and leave bad impression. # Input -Input to a command is provided via `ARGUMENTS` and `OPTIONS`. +Input to a command is provided via `ARGUMENTS` and `OPTIONS`. `ARGUMENTS` represent a required input for a function. When choosing to use `ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it: @@ -302,7 +302,7 @@ $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -342,7 +342,7 @@ also allowing them to redirect content to a file. For example: ```shell $ nix build > build.txt ------------------------------------------------------------------------ - Error! Atrribute `bin` missing at (1:94) from string. + Error! Attribute `bin` missing at (1:94) from string. ------------------------------------------------------------------------ 1| with import { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } "" @@ -408,7 +408,7 @@ Above command clearly states that command successfully completed. And in case of `nix build`, which is a command that might take some time to complete, it is equally important to also show that a command started. -## Text alignment +## Text alignment Text alignment is the number one design element that will present all of the Nix commands as a family and not as separate tools glued together. @@ -419,7 +419,7 @@ The format we should follow is: $ nix COMMAND VERB_1 NOUN and other words VERB__1 NOUN and other words - |> Some details + |> Some details ``` Few rules that we can extract from above example: @@ -444,13 +444,13 @@ is not even notable, therefore relying on it wouldn’t make much sense. **The bright text is much better supported** across terminals and color schemes. Most of the time the difference is perceived as if the bright text -would be bold. +would be bold. ## Colors Humans are already conditioned by society to attach certain meaning to certain colors. While the meaning is not universal, a simple collection of colors is -used to represent basic emotions. +used to represent basic emotions. Colors that can be used in output @@ -555,7 +555,7 @@ $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -566,7 +566,7 @@ $ nix build --option substitutors https://cache.example.org There are many ways that you can control verbosity. -Verbosity levels are: +Verbosity levels are: - `ERROR` (level 0) - `WARN` (level 1) @@ -586,4 +586,4 @@ There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and # Appendix 1: Commands naming exceptions -`nix init` and `nix repl` are well established +`nix init` and `nix repl` are well established diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md index 87127de2a..c631a8453 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/expressions/builtins-prefix.md @@ -12,5 +12,5 @@ For instance, `derivation` is also available as `builtins.derivation`.
derivation attrs; builtins.derivation attrs
-

derivation in described in +

derivation is described in its own section.

diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/src/package-management/garbage-collection.md index fecb30fd6..29a3b3101 100644 --- a/doc/manual/src/package-management/garbage-collection.md +++ b/doc/manual/src/package-management/garbage-collection.md @@ -44,7 +44,7 @@ collector as follows: $ nix-store --gc ``` -The behaviour of the gargage collector is affected by the +The behaviour of the garbage collector is affected by the `keep-derivations` (default: true) and `keep-outputs` (default: false) options in the Nix configuration file. The defaults will ensure that all derivations that are build-time dependencies of garbage collector roots From 79d07d0980e7b19e613ebbfac420fe2aaaed6469 Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Sun, 14 Nov 2021 12:23:46 +0100 Subject: [PATCH 47/85] libfetchers: set free gitlab headers --- src/libfetchers/github.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ffc44e9e2..1c539b80e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme if ("PAT" == token.substr(0, fldsplit)) return std::make_pair("Private-token", token.substr(fldsplit+1)); warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); - return std::nullopt; + return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } Hash getRevFromRef(nix::ref store, const Input & input) const override From 1d0bc96c96aff1b327c46c6aa6a2ff9133a70a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 15 Oct 2021 18:41:50 +0000 Subject: [PATCH 48/85] Add backport action --- .github/workflows/backport.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/backport.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..af12190e2 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,26 @@ +name: Backport +on: + pull_request_target: + types: [closed, labeled] +jobs: + backport: + name: Backport Pull Request + if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + # required to find all branches + fetch-depth: 0 + - name: Create backport PRs + # should be kept in sync with `version` + uses: zeebe-io/backport-action@v0.0.5 + with: + # Config README: https://github.com/zeebe-io/backport-action#backport-action + github_token: ${{ secrets.GITHUB_TOKEN }} + github_workspace: ${{ github.workspace }} + pull_description: |- + Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + # should be kept in sync with `uses` + version: v0.0.5 From 671817a858875045b5f99a15ac92addaf9b207f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Nov 2021 18:44:27 +0100 Subject: [PATCH 49/85] Simplify lockFlake() a bit --- src/libexpr/flake/flake.cc | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 1dc6f5694..f5be67d67 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -446,24 +446,18 @@ LockedFlake lockFlake( update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath); - } else { + FlakeInputs fakeInputs; + + if (!mustRefetch) { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ - FlakeInputs fakeInputs; - - bool refetch = false; - for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -475,7 +469,7 @@ LockedFlake lockFlake( // If the override disappeared, we have to refetch the flake, // since some of the inputs may not be present in the lockfile. if (o == input.overrides.end()) { - refetch = true; + mustRefetch = true; // There's no point populating the rest of the fake inputs, // since we'll refetch the flake anyways. break; @@ -485,13 +479,14 @@ LockedFlake lockFlake( }); } } - - if (refetch) - fakeInputs = getFlake(state, oldLock->lockedRef, false, flakeCache).inputs; - - computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); } + computeLocks( + mustRefetch + ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs + : fakeInputs, + childNode, inputPath, oldLock, parent, parentPath); + } else { /* We need to create a new lock file entry. So fetch this input. */ From 4ba355e593e220e5ef6d9d6dfc78990868a724e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 22:01:13 +0000 Subject: [PATCH 50/85] Bump cachix/install-nix-action from 14.1 to 15 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 14.1 to 15. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v14.1...v15) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 492ee5388..be4fcd8d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v14.1 + - uses: cachix/install-nix-action@v15 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 if: needs.check_cachix.outputs.secret == 'true' @@ -50,7 +50,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v14.1 + - uses: cachix/install-nix-action@v15 - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' @@ -69,7 +69,7 @@ jobs: steps: - uses: actions/checkout@v2.4.0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v14.1 + - uses: cachix/install-nix-action@v15 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" From 3771f931bfc5902047755ef810bed299df91400d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 22:01:17 +0000 Subject: [PATCH 51/85] Bump zeebe-io/backport-action from 0.0.5 to 0.0.7 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 0.0.5 to 0.0.7. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v0.0.5...v0.0.7) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index af12190e2..ec7ab4516 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.5 + uses: zeebe-io/backport-action@v0.0.7 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 8c93a481af2ce8fbcdb9e2bbcc9559d52703112f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2021 14:23:05 +0100 Subject: [PATCH 52/85] Ignore errors unsharing/restoring the mount namespace This prevents Nix from barfing when run in a container where it doesn't have the appropriate privileges. --- src/libutil/util.cc | 14 ++++++++++---- src/nix/main.cc | 8 +++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a6552ebca..8ae3445c6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1631,6 +1631,7 @@ void setStackSize(size_t stackSize) } #endif } + static AutoCloseFD fdSavedMountNamespace; void saveMountNamespace() @@ -1638,9 +1639,10 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); }); #endif } @@ -1648,8 +1650,12 @@ void saveMountNamespace() void restoreMountNamespace() { #if __linux__ - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } #endif } diff --git a/src/nix/main.cc b/src/nix/main.cc index 01889a71f..60b0aa410 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -257,9 +257,11 @@ void mainWrapped(int argc, char * * argv) #if __linux__ if (getuid() == 0) { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } catch (Error & e) { } } #endif From 8368a8aff1b70e98a4997d803e34bfdc2acf10f4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Nov 2021 10:32:26 -0500 Subject: [PATCH 53/85] Make docker.nix match Nixpkgs's idioms 1. `target` is the wrong name, that is just for compilers per out standard terminology. We just need to worry about "build" and "host". 2. We only need one `pkgs`. `pkgs.buildPackages` is how we get anything we need at build time. 3. `crossSystem` is the name of a nixpkgs parameter that is actually an attribute set, not a 2-part "cpu-os" string. 3. `pkgsCross` effectively evaluates Nixpkgs twice, which is inefficient. It is just there for people poking around the CLI / REPL (and I am skeptical even that is a good idea), and *not* what written code should use, especially code that is merely parametric in the package set it is given. 4. We don't need to memoize Nixpkgs here because we are only doing one pkg set at a time (no `genAttrs`) so it's better to just delete all this stuff. `flake.nix` instead would do something like that, with `genAttrs` (though without `pkgsCross`), if and when we have hydra jobs for cross builds. --- docker.nix | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/docker.nix b/docker.nix index 316d57a36..2a13c23fb 100644 --- a/docker.nix +++ b/docker.nix @@ -2,24 +2,11 @@ , lib ? pkgs.lib , name ? "nix" , tag ? "latest" -, crossSystem ? null , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" }: let - buildPkgs = pkgs; - targetPkgs = - if crossSystem != null && crossSystem != pkgs.system - then { - aarch64-linux = pkgs.pkgsCross.aarch64-multiplatform; - armv7l-linux = pkgs.pkgsCross.armv7l-hf-multiplatform.system; - x86_64-linux = pkgs.pkgsCross.gnu64; - powerpc64le-linux = pkgs.pkgsCross.musl-power; - i686-linux = pkgs.pkgsCross.gnu32; - }.${crossSystem} - else pkgs; - - defaultPkgs = with targetPkgs; [ + defaultPkgs = with pkgs; [ nix bashInteractive coreutils-full @@ -140,17 +127,17 @@ let baseSystem = let - nixpkgs = targetPkgs.path; - channel = targetPkgs.runCommand "channel-nixos" { } '' + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixos" { } '' mkdir $out ln -s ${nixpkgs} $out/nixpkgs echo "[]" > $out/manifest.nix ''; - rootEnv = pkgs.buildEnv { + rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; paths = defaultPkgs; }; - profile = targetPkgs.runCommand "user-environment" { } '' + profile = pkgs.buildPackages.runCommand "user-environment" { } '' mkdir $out cp -a ${rootEnv}/* $out/ @@ -175,7 +162,7 @@ let EOF ''; in - targetPkgs.runCommand "base-system" + pkgs.runCommand "base-system" { inherit passwdContents groupContents shadowContents nixConfContents; passAsFile = [ @@ -225,12 +212,12 @@ let echo "${channelURL} ${channelName}" > $out/root/.nix-channels mkdir -p $out/bin $out/usr/bin - ln -s ${targetPkgs.coreutils}/bin/env $out/usr/bin/env - ln -s ${targetPkgs.bashInteractive}/bin/bash $out/bin/sh + ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env + ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh ''; in -targetPkgs.dockerTools.buildLayeredImageWithNixDb { +pkgs.dockerTools.buildLayeredImageWithNixDb { inherit name tag; From e41cf8511f68e7e67786d8735d03c961c34cb2ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2021 17:44:19 +0100 Subject: [PATCH 54/85] Don't hang when calling an attrset Fixes #5565. --- src/libexpr/eval.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 402de78ad..615f020e4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1275,6 +1275,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } }; + Attr * functor; + while (nrArgs > 0) { if (vCur.isLambda()) { @@ -1403,16 +1405,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } } - else if (vCur.type() == nAttrs) { - if (auto functor = vCur.attrs->get(sFunctor)) { - /* 'vCur" may be allocated on the stack of the calling - function, but for functors we may keep a reference, - so heap-allocate a copy and use that instead. */ - Value * args2[] = {allocValue()}; - *args2[0] = vCur; - /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 1, args2, vCur, pos); - } + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur" may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue()}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 1, args2, vCur, pos); } else From d7bae52b9d418e208593c6641bce54f6a82458f4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2021 22:34:17 +0100 Subject: [PATCH 55/85] Call functors with both arguments at once This is not really useful on its own, but it does recover the 'infinite recursion' error message for '{ __functor = x: x; } 1', and is more efficient in conjunction with #3718. Fixes #5515. --- src/libexpr/eval.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 615f020e4..f1ff3a6e0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1406,13 +1406,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { - /* 'vCur" may be allocated on the stack of the calling + /* 'vCur' may be allocated on the stack of the calling function, but for functors we may keep a reference, so heap-allocate a copy and use that instead. */ - Value * args2[] = {allocValue()}; + Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 1, args2, vCur, pos); + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; } else From 6d46b5b609f8f85968e1ca7aaf7c57dd52d0521c Mon Sep 17 00:00:00 2001 From: Kalle Jepsen Date: Wed, 17 Nov 2021 08:41:26 +0100 Subject: [PATCH 56/85] Fix detection of scp-style URIs to support non-standard SSH ports for git --- src/libexpr/primops/fetchTree.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6becdafc..b307ac04a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -74,8 +74,11 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau std::string fixURIForGit(std::string uri, EvalState & state) { + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ static std::regex scp_uri("([^/].*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) + if (uri[0] != '/' && std::regex_match(uri, scp_uri) && uri.find("://") == std::string::npos) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else return fixURI(uri, state); From d03e89e5d19a7082cc1e8e5ff10d7f62a0a6e937 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Mar 2020 15:32:20 +0100 Subject: [PATCH 57/85] Parse '(f x) y' the same as 'f x y' (cherry picked from commit 5253cb4b68ad248f37b27849c0ebf3614e4f2777) --- src/libexpr/parser.y | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 923997bf6..c1f4e72e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -39,12 +39,6 @@ namespace nix { { }; }; - // Helper to prevent an expensive dynamic_cast call in expr_app. - struct App - { - Expr * e; - bool isCall; - }; } #define YY_DECL int yylex \ @@ -284,12 +278,10 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err char * uri; std::vector * attrNames; std::vector * string_parts; - nix::App app; // bool == whether this is an ExprCall } %type start expr expr_function expr_if expr_op -%type expr_select expr_simple -%type expr_app +%type expr_select expr_simple expr_app %type expr_list %type binds %type formals @@ -377,20 +369,18 @@ expr_op | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } - | expr_app { $$ = $1.e; } + | expr_app ; expr_app : expr_app expr_select { - if ($1.isCall) { - ((ExprCall *) $1.e)->args.push_back($2); + if (auto e2 = dynamic_cast($1)) { + e2->args.push_back($2); $$ = $1; - } else { - $$.e = new ExprCall(CUR_POS, $1.e, {$2}); - $$.isCall = true; - } + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); } - | expr_select { $$.e = $1; $$.isCall = false; } + | expr_select ; expr_select From 46d2a5a10be7e48679a29d487adbb6f1d6fd452a Mon Sep 17 00:00:00 2001 From: Kalle Jepsen Date: Wed, 17 Nov 2021 13:49:10 +0100 Subject: [PATCH 58/85] Simplify fix by disallowing / in front of @ to match scp style --- src/libexpr/primops/fetchTree.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b307ac04a..079513873 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -77,8 +77,8 @@ std::string fixURIForGit(std::string uri, EvalState & state) /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes * them by removing the `:` and assuming a scheme of `ssh://` * */ - static std::regex scp_uri("([^/].*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri) && uri.find("://") == std::string::npos) + static std::regex scp_uri("([^/]*)@(.*):(.*)"); + if (uri[0] != '/' && std::regex_match(uri, scp_uri)) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else return fixURI(uri, state); From e96faadcd6d0adc223deca746a64f5ae46b517f0 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 17 Nov 2021 14:28:43 +0100 Subject: [PATCH 59/85] Fix XDG_CONFIG_DIRS fallback According to XDG Base Directory Specification, it should fall back to /etc/xdg when the env var is not present. --- doc/manual/src/command-ref/conf-file-prefix.md | 5 +++-- src/libutil/util.cc | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index d660db502..44b7ba86d 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -16,8 +16,9 @@ By default Nix reads settings from the following places: will be loaded in reverse order. Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If these are unset, it will look in - `$HOME/.config/nix/nix.conf`. + and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to + `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` + as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8ae3445c6..5468d1ed1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -562,7 +562,7 @@ Path getConfigDir() std::vector getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector result = tokenizeString>(configDirs, ":"); result.insert(result.begin(), configHome); return result; From ca4d8ce9e22ef04687b1516c9472a5d3f82e056b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Wed, 17 Nov 2021 16:30:39 +0100 Subject: [PATCH 60/85] doc: De-emphasize nix-env without -A The manual uses `nix-env -i` without `-A` prominently, teaching a bad practice to newcomers. --- doc/manual/src/command-ref/nix-env.md | 42 ++++++++----- doc/manual/src/command-ref/opt-common.md | 4 +- .../expressions/simple-building-testing.md | 2 +- doc/manual/src/introduction.md | 4 +- .../package-management/basic-package-mgmt.md | 63 ++++++++++--------- .../binary-cache-substituter.md | 4 +- doc/manual/src/package-management/profiles.md | 6 +- .../src/package-management/ssh-substituter.md | 2 +- doc/manual/src/quick-start.md | 14 ++--- 9 files changed, 80 insertions(+), 61 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 5217510d7..8d6abaf52 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -238,7 +238,16 @@ a number of possible ways: ## Examples -To install a specific version of `gcc` from the active Nix expression: +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env -iA gcc40mips +installing `gcc-4.0.2' +$ nix-env -iA xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: ```console $ nix-env --install gcc-3.3.2 @@ -246,6 +255,9 @@ installing `gcc-3.3.2' uninstalling `gcc-3.1' ``` +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + Note the previously installed version is removed, since `--preserve-installed` was not specified. @@ -256,13 +268,6 @@ $ nix-env --install gcc installing `gcc-3.3.2' ``` -To install using a specific attribute: - -```console -$ nix-env -i -A gcc40mips -$ nix-env -i -A xorg.xorgserver -``` - To install all derivations in the Nix expression `foo.nix`: ```console @@ -374,22 +379,29 @@ For the other flags, see `--install`. ## Examples ```console -$ nix-env --upgrade gcc +$ nix-env --upgrade -A nixpkgs.gcc upgrading `gcc-3.3.1' to `gcc-3.4' ``` +When there are no updates available, nothing will happen: + ```console -$ nix-env -u gcc-3.3.2 --always (switch to a specific version) +$ nix-env --upgrade -A nixpkgs.pan +``` + +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + +```console +$ nix-env -u gcc-3.3.2 --always upgrading `gcc-3.4' to `gcc-3.3.2' ``` -```console -$ nix-env --upgrade pan -(no upgrades available, so nothing happens) -``` +To try to upgrade everything +(matching packages based on the part of the derivation name without version): ```console -$ nix-env -u (try to upgrade everything) +$ nix-env -u upgrading `hello-2.1.2' to `hello-2.1.3' upgrading `mozilla-1.2' to `mozilla-1.4' ``` diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 47862bc09..7ee1a26bc 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options: }: ... ``` - So if you call this Nix expression (e.g., when you do `nix-env -i + So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](../expressions/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., - `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that + `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md index 6f730a936..7f0d8f841 100644 --- a/doc/manual/src/expressions/simple-building-testing.md +++ b/doc/manual/src/expressions/simple-building-testing.md @@ -1,6 +1,6 @@ # Building and Testing -You can now try to build Hello. Of course, you could do `nix-env -i +You can now try to build Hello. Of course, you could do `nix-env -f . -iA hello`, but you may not want to install a possibly broken package just yet. The best way to test the package is by using the command `nix-build`, which builds a Nix expression and creates a symlink named diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index 93ec502c1..d87487a07 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -76,7 +76,7 @@ there after an upgrade. This means that you can _roll back_ to the old version: ```console -$ nix-env --upgrade some-packages +$ nix-env --upgrade -A nixpkgs.some-package $ nix-env --rollback ``` @@ -122,7 +122,7 @@ Nix expressions generally describe how to build a package from source, so an installation action like ```console -$ nix-env --install firefox +$ nix-env --install -A nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 9702a29eb..50c6d3c2d 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs, or completely new ones.) You can manually download the latest version of Nixpkgs from -. However, it’s much more +. However, it’s much more convenient to use the Nixpkgs [*channel*](channels.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you @@ -47,41 +47,45 @@ $ nix-channel --update You can view the set of available packages in Nixpkgs: ```console -$ nix-env -qa -aterm-2.2 -bash-3.0 -binutils-2.15 -bison-1.875d -blackdown-1.4.2 -bzip2-1.0.2 +$ nix-env -qaP +nixpkgs.aterm aterm-2.2 +nixpkgs.bash bash-3.0 +nixpkgs.binutils binutils-2.15 +nixpkgs.bison bison-1.875d +nixpkgs.blackdown blackdown-1.4.2 +nixpkgs.bzip2 bzip2-1.0.2 … ``` -The flag `-q` specifies a query operation, and `-a` means that you want +The flag `-q` specifies a query operation, `-a` means that you want to show the “available” (i.e., installable) packages, as opposed to the -installed packages. If you downloaded Nixpkgs yourself, or if you -checked it out from GitHub, then you need to pass the path to your -Nixpkgs tree using the `-f` flag: +installed packages, and `-P` prints the attribute paths that can be used +to unambiguously select a package for installation (listed in the first column). +If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the `-f` flag: ```console -$ nix-env -qaf /path/to/nixpkgs +$ nix-env -qaPf /path/to/nixpkgs +aterm aterm-2.2 +bash bash-3.0 +… ``` where */path/to/nixpkgs* is where you’ve unpacked or checked out Nixpkgs. -You can select specific packages by name: +You can filter the packages by name: ```console -$ nix-env -qa firefox -firefox-34.0.5 -firefox-with-plugins-34.0.5 +$ nix-env -qaP firefox +nixpkgs.firefox-esr firefox-91.3.0esr +nixpkgs.firefox firefox-94.0.1 ``` and using regular expressions: ```console -$ nix-env -qa 'firefox.*' +$ nix-env -qaP 'firefox.*' ``` It is also possible to see the *status* of available packages, i.e., @@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in the system: ```console -$ nix-env -qas +$ nix-env -qaPs … --PS bash-3.0 ---S binutils-2.15 -IPS bison-1.875d +-PS nixpkgs.bash bash-3.0 +--S nixpkgs.binutils binutils-2.15 +IPS nixpkgs.bison bison-1.875d … ``` @@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally. -You can install a package using `nix-env -i`. For instance, +You can install a package using `nix-env -iA`. For instance, ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` -will install the package called `subversion` (which is, of course, the +will install the package called `subversion` from `nixpkgs` channel (which is, of course, the [Subversion version management system](http://subversion.tigris.org/)). > **Note** @@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the > binary cache ; it contains binaries for most > packages in Nixpkgs. Only if no binary is available in the binary > cache, Nix will build the package from source. So if `nix-env -> -i subversion` results in Nix building stuff from source, then either +> -iA nixpkgs.subversion` results in Nix building stuff from source, then either > the package is not built for your platform by the Nixpkgs build > servers, or your version of Nixpkgs is too old or too new. For > instance, if you have a very recent checkout of Nixpkgs, then the @@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the > using a Git checkout of the Nixpkgs tree), you will get binaries for > most packages. -Naturally, packages can also be uninstalled: +Naturally, packages can also be uninstalled. Unlike when installing, you will +need to use the derivation name (though the version part can be omitted), +instead of the attribute path, as `nix-env` does not record which attribute +was used for installing: ```console $ nix-env -e subversion @@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of Nix Packages, you can do: ```console -$ nix-env -u subversion +$ nix-env -uA nixpkgs.subversion ``` This will *only* upgrade Subversion if there is a “newer” version in the diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md index bdc5038fc..ef738794b 100644 --- a/doc/manual/src/package-management/binary-cache-substituter.md +++ b/doc/manual/src/package-management/binary-cache-substituter.md @@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -i nix-serve +$ nix-env -iA nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using `--option extra-binary-caches`, e.g.: ```console -$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ +$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/ ``` The option `extra-binary-caches` tells Nix to use this binary cache in diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md index fbbfb7320..d1a2580d4 100644 --- a/doc/manual/src/package-management/profiles.md +++ b/doc/manual/src/package-management/profiles.md @@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be what we would obtain if we had done ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ environment is generated based on the current one. For instance, generation 43 was created from generation 42 when we did ```console -$ nix-env -i subversion firefox +$ nix-env -iA nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion +$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md index 6e5e258bc..c59933f61 100644 --- a/doc/manual/src/package-management/ssh-substituter.md +++ b/doc/manual/src/package-management/ssh-substituter.md @@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are available on the server `avalon`: ```console -$ nix-env -i firefox --substituters ssh://alice@avalon +$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 71205923b..b54e73500 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,19 +19,19 @@ to subsequent chapters. channel: ```console - $ nix-env -qa - docbook-xml-4.3 - docbook-xml-4.5 - firefox-33.0.2 - hello-2.9 - libxslt-1.1.28 + $ nix-env -qaP + nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 + nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 + nixpkgs.firefox firefox-33.0.2 + nixpkgs.hello hello-2.9 + nixpkgs.libxslt libxslt-1.1.28 … ``` 1. Install some packages from the channel: ```console - $ nix-env -i hello + $ nix-env -iA nixpkgs.hello ``` This should download pre-built packages; it should not build them From bc14465e08a286f8bc3b1e47e70372352b64bd2c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Thu, 18 Nov 2021 04:00:19 +0000 Subject: [PATCH 61/85] Fix stack buffer overflow Fix a stack buffer overflow found by running MemorySanitizer. --- src/libstore/references.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index c369b14ac..bb9590acb 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,7 +54,7 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), refLength); + s.append(data.data(), std::min(data.size(), refLength)); search(s, hashes, seen); search(data, hashes, seen); From 9653858ce68a04e2c23dbf14e344d4dca4bf7e24 Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Wed, 17 Nov 2021 16:38:03 +0100 Subject: [PATCH 62/85] Fix :e in repl Closes https://github.com/NixOS/nix/issues/5487 Co-authored-by: Alexander Bantyev balsoft@balsoft.ru --- src/nix/repl.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 4f13ee05d..fd86174f2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -471,7 +471,10 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, true, args); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); // Reload right after exiting the editor state->resetFileCache(); From 262a3c7ce332d8f7f52b007178c2d54f063d6c7a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Nov 2021 12:12:31 +0100 Subject: [PATCH 63/85] Simplify --- src/libstore/references.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index bb9590acb..91b3fc142 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), std::min(data.size(), refLength)); + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); search(s, hashes, seen); search(data, hashes, seen); - auto tailLen = std::min(data.size(), refLength); auto rest = refLength - tailLen; if (rest < tail.size()) tail = tail.substr(tail.size() - rest); From 79f27500a45f9b1eda3d54114cf835f73775af8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Nov 2021 13:32:52 +0100 Subject: [PATCH 64/85] Test that untrusted config is ignored without --accept-flake-config --- tests/flake-local-settings.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/flake-local-settings.sh b/tests/flake-local-settings.sh index 6ee6b17d6..09f6b4ca8 100644 --- a/tests/flake-local-settings.sh +++ b/tests/flake-local-settings.sh @@ -25,5 +25,10 @@ cat < flake.nix } EOF +# Without --accept-flake-config, the post hook should not run. +nix build < /dev/null +(! [[ -f post-hook-ran ]]) +clearStore + nix build --accept-flake-config test -f post-hook-ran || fail "The post hook should have ran" From e1192116d357fd8346ebeed64de25730e47b4147 Mon Sep 17 00:00:00 2001 From: Lorenz Leutgeb Date: Thu, 18 Nov 2021 15:08:01 +0100 Subject: [PATCH 65/85] doc: Fix escape for operator "logical or" See https://matrix.to/#/!KqkRjyTEzAGRiZFBYT:nixos.org/$hhMb6AdRIXfRkv_LsNsiQJuch7AQ_b6szr4tfawFy-4 --- doc/manual/src/expressions/language-operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index b7fd6f4c6..6ab319039 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -24,5 +24,5 @@ order of precedence (from strongest to weakest binding). | Equality | *e1* `==` *e2* | none | Equality. | 11 | | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | -| Logical OR | *e1* `\|\|` *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| e2`). | 14 | +| Logical OR | *e1* || *e2* | left | Logical OR. | 13 | +| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to !e1 || e2). | 14 | From 9de324f554b884db7c243836b9294b8cca811fc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Nov 2021 21:17:57 +0100 Subject: [PATCH 66/85] Remove nix-reduce-build and nix-http-export These scripts are not installed and haven't been updated in many years. --- .gitignore | 2 - scripts/local.mk | 4 +- scripts/nix-http-export.cgi.in | 51 ---------- scripts/nix-reduce-build.in | 171 --------------------------------- 4 files changed, 1 insertion(+), 227 deletions(-) delete mode 100755 scripts/nix-http-export.cgi.in delete mode 100755 scripts/nix-reduce-build.in diff --git a/.gitignore b/.gitignore index b6a5f6adc..2889a56eb 100644 --- a/.gitignore +++ b/.gitignore @@ -26,8 +26,6 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-reduce-build -/scripts/nix-http-export.cgi /scripts/nix-profile-daemon.sh # /src/libexpr/ diff --git a/scripts/local.mk b/scripts/local.mk index 2a0055852..b8477178e 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,7 +1,5 @@ nix_noinst_scripts := \ - $(d)/nix-http-export.cgi \ - $(d)/nix-profile.sh \ - $(d)/nix-reduce-build + $(d)/nix-profile.sh noinst-scripts += $(nix_noinst_scripts) diff --git a/scripts/nix-http-export.cgi.in b/scripts/nix-http-export.cgi.in deleted file mode 100755 index 19a505af1..000000000 --- a/scripts/nix-http-export.cgi.in +++ /dev/null @@ -1,51 +0,0 @@ -#! /bin/sh - -export HOME=/tmp -export NIX_REMOTE=daemon - -TMP_DIR="${TMP_DIR:-/tmp/nix-export}" - -@coreutils@/mkdir -p "$TMP_DIR" || true -@coreutils@/chmod a+r "$TMP_DIR" - -needed_path="?$QUERY_STRING" -needed_path="${needed_path#*[?&]needed_path=}" -needed_path="${needed_path%%&*}" -#needed_path="$(echo $needed_path | ./unhttp)" -needed_path="${needed_path//%2B/+}" -needed_path="${needed_path//%3D/=}" - -echo needed_path: "$needed_path" >&2 - -NIX_STORE="${NIX_STORE_DIR:-/nix/store}" - -echo NIX_STORE: "${NIX_STORE}" >&2 - -full_path="${NIX_STORE}"/"$needed_path" - -if [ "$needed_path" != "${needed_path%.drv}" ]; then - echo "Status: 403 You should create the derivation file yourself" - echo "Content-Type: text/plain" - echo - echo "Refusing to disclose derivation contents" - exit -fi - -if @bindir@/nix-store --check-validity "$full_path"; then - if ! [ -e nix-export/"$needed_path".nar.gz ]; then - @bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz - @coreutils@/ln -fs "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz - fi; - echo "Status: 301 Moved" - echo "Location: nix-export/"$needed_path".nar.gz" - echo -else - echo "Status: 404 No such path found" - echo "Content-Type: text/plain" - echo - echo "Path not found:" - echo "$needed_path" - echo "checked:" - echo "$full_path" -fi - diff --git a/scripts/nix-reduce-build.in b/scripts/nix-reduce-build.in deleted file mode 100755 index 50beb9d10..000000000 --- a/scripts/nix-reduce-build.in +++ /dev/null @@ -1,171 +0,0 @@ -#! @bash@ - -WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX); -cd "$WORKING_DIRECTORY"; - -if test -z "$1" || test "a--help" = "a$1" ; then - echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2 - echo As in: >&2 - echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2 - echo nix-reduce-build /etc/nixos/nixos -- \\ - echo " " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2 - echo " store path name will be added into the end of the URL" >&2 - echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2 - echo " that should be a directory where gzipped 'nix-store --export' ">&2 - echo " files are located (they should have .nar.gz extension)" >&2 - echo " Or all together: " >&2 - echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\ - " ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2 - echo " Also supports best-effort local builds of failing expression set:" >&2 - echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2 - echo " nix-daemon:// builds using daemon" - echo " nix-self:// builds directly using nix-store from current installation" >&2 - echo " nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2; - echo "derivations with specified output hash (sha256, sha1 or md5)." >&2 - echo " nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2; - echo "maximum amount of paths" >&2; - echo " nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2; - echo "maximum amount of paths" >&2; - echo " If no package sources are specified, required paths are listed." >&2; - exit; -fi; - -while ! test "$1" = "--" || test "$1" = "" ; do - echo "$1" >> initial; >&2 - shift; -done -shift; -echo Will work on $(cat initial | wc -l) targets. >&2 - -while read ; do - case "$REPLY" in - ${NIX_STORE_DIR:-/nix/store}/*) - echo "$REPLY" >> paths; >&2 - ;; - *) - ( - IFS=: ; - nix-instantiate $REPLY >> paths; - ); - ;; - esac; -done < initial; -echo Proceeding $(cat paths | wc -l) paths. >&2 - -while read; do - case "$REPLY" in - *.drv) - echo "$REPLY" >> derivers; >&2 - ;; - *) - nix-store --query --deriver "$REPLY" >>derivers; - ;; - esac; -done < paths; -echo Found $(cat derivers | wc -l) derivers. >&2 - -cat derivers | xargs nix-store --query -R > derivers-closure; -echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2 - -cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths; -cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths; -echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2 - -cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; -echo We need $(cat needed-paths | wc -l) paths. >&2 - -egrep '[.]drv$' derivers-closure > critical-derivers; - -if test -z "$1" ; then - cat needed-paths; -fi; - -refresh_critical_derivers() { - echo "Finding needed derivers..." >&2; - cat critical-derivers | while read; do - if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then - echo "$REPLY"; - fi; - done > new-critical-derivers; - mv new-critical-derivers critical-derivers; - echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2 -} - -build_here() { - cat critical-derivers | while read; do - echo "Realising $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${REPLY}" - done; -} - -try_to_substitute(){ - cat needed-paths | while read ; do - echo "Building $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}" - done; -} - -for i in "$@"; do - sshHost="${i#ssh://}"; - httpHost="${i#http://}"; - httpsHost="${i#https://}"; - filePath="${i#file:/}"; - if [ "$i" != "$sshHost" ]; then - cat needed-paths | while read; do - echo "Getting $REPLY and its closure over ssh" >&2 - nix-copy-closure --from "$sshHost" --gzip "$REPLY" &2 - curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import; - done; - elif [ "$i" != "$filePath" ] ; then - cat needed-paths | while read; do - echo "Installing $REPLY from file" >&2 - gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import; - done; - elif [ "$i" = "nix-daemon://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self://" ] ; then - NIX_REMOTE= try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE= build_here; - elif [ "$i" = "nix-daemon-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using nix-daemon" >&2 - NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-self-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using direct Nix build" >&2 - NIX_REMOTE= @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-daemon-substitute://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - elif [ "$i" = "nix-self-substitute://" ] ; then - NIX_REMOTE= try_to_substitute; - elif [ "$i" = "nix-daemon-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE= build_here; - fi; - mv needed-paths wanted-paths; - cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; - echo We still need $(cat needed-paths | wc -l) paths. >&2 -done; - -cd / -rm -r "$WORKING_DIRECTORY" From 06fb6aecea43218e66ccc879471ffebb7dbfee78 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Nov 2021 22:22:33 +0000 Subject: [PATCH 67/85] Fix testing the other daemon The eventual PATH entry needs the `.../bin` or we will not use the right daemon. --- tests/common.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 3d0d56120..61abab1d7 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -38,7 +38,7 @@ if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then fi DAEMON_PATH="$PATH" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}:$DAEMON_PATH" + DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" fi coreutils=@coreutils@ From d5b36bdb58c5b8182b36ee6869936323a2f2fec7 Mon Sep 17 00:00:00 2001 From: Kristof Molnar-Tatai Date: Fri, 19 Nov 2021 13:10:04 +0100 Subject: [PATCH 68/85] switch order of wget and curl This change makes the script consistent with the installation instructions while keeping wget as an alternative. --- scripts/install.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 5be4f9dda..38d1fb36f 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -81,10 +81,10 @@ if [ "$(uname -s)" != "Darwin" ]; then require_util xz "unpack the binary tarball" fi -if command -v wget > /dev/null 2>&1; then - fetch() { wget "$1" -O "$2"; } -elif command -v curl > /dev/null 2>&1; then +if command -v curl > /dev/null 2>&1; then fetch() { curl -L "$1" -o "$2"; } +elif command -v wget > /dev/null 2>&1; then + fetch() { wget "$1" -O "$2"; } else oops "you don't have wget or curl installed, which I need to download the binary tarball" fi From eff48e84d9fde4e98adcdd0ff325b93a8d47947a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Thu, 18 Nov 2021 18:33:13 +0100 Subject: [PATCH 69/85] Apply some shellcheck suggestions --- scripts/install-multi-user.sh | 2 +- scripts/install-nix-from-closure.sh | 14 +++++++------- scripts/install-systemd-multi-user.sh | 2 +- scripts/nix-profile-daemon.sh.in | 6 +++--- scripts/prepare-installer-for-github-actions | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index eba894799..0dba36f51 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -818,7 +818,7 @@ main() { # can fail faster in this case. Sourcing install-darwin... now runs # `touch /` to detect Read-only root, but it could update times on # pre-Catalina macOS if run as root user. - if [ $EUID -eq 0 ]; then + if [ "$EUID" -eq 0 ]; then failure <&2 -if ! [ -e $dest ]; then +if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then @@ -143,12 +143,12 @@ if ! [ -e $dest ]; then fi fi -if ! [ -w $dest ]; then +if ! [ -w "$dest" ]; then echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi -mkdir -p $dest/store +mkdir -p "$dest/store" printf "copying Nix to %s..." "${dest}/store" >&2 # Insert a newline if no progress is shown. @@ -189,17 +189,17 @@ fi # Install an SSL certificate bundle. if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then - $nix/bin/nix-env -i "$cacert" + "$nix/bin/nix-env" -i "$cacert" export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" fi # Subscribe the user to the Nixpkgs channel and fetch it. if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then - if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + if ! "$nix/bin/nix-channel" --list | grep -q "^nixpkgs "; then + "$nix/bin/nix-channel" --add https://nixos.org/channels/nixpkgs-unstable fi if [ -z "$_NIX_INSTALLER_TEST" ]; then - if ! $nix/bin/nix-channel --update nixpkgs; then + if ! "$nix/bin/nix-channel" --update nixpkgs; then echo "Fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"nix-channel --update nixpkgs\"." fi diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 81c61b2a0..f4a2dfc5d 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -15,7 +15,7 @@ readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf create_systemd_override() { header "Configuring proxy for the nix-daemon service" - _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname "$SERVICE_OVERRIDE")" cat < Date: Fri, 19 Nov 2021 15:22:31 +0100 Subject: [PATCH 70/85] Fix build warnings on MacOS --- src/libstore/build/local-derivation-goal.cc | 5 ++++- src/libstore/gc.cc | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3c7bd695e..c9a4a31e7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -342,7 +342,7 @@ int childEntry(void * arg) return 1; } - +#if __linux__ static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -358,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to) copyPath(from, to); } } +#endif void LocalDerivationGoal::startBuilder() @@ -917,7 +918,9 @@ void LocalDerivationGoal::startBuilder() } else #endif { +#if __linux__ fallback: +#endif pid = startProcess([&]() { runChild(); }); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8030c84f5..29d45e067 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -324,6 +324,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } +#if defined(__linux__) static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -333,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots) throw; } } +#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { From 4318ba2ec568a9fe7d5b4b014df6b7d252ae3481 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 19 Nov 2021 23:52:52 -0500 Subject: [PATCH 71/85] add real path to allowedPaths --- src/libexpr/eval.cc | 4 ++-- src/libexpr/primops.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f1ff3a6e0..97fc04711 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -520,7 +520,7 @@ Path EvalState::checkSourcePath(const Path & path_) } if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + throw RestrictedPathError("access to absolute path '%1%' is forbidden in restricted mode", abspath); /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); @@ -533,7 +533,7 @@ Path EvalState::checkSourcePath(const Path & path_) } } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); + throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5bd4e5545..8dccf6401 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -70,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context) if (outputPaths.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store->printStorePath(drvPath), outputName); - allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + allowPath(outputPaths.at(outputName)); } } } From 664ee49e0d6f1ce0a1608559beeaf82bd039c690 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Sun, 21 Nov 2021 15:52:01 +0000 Subject: [PATCH 72/85] nix-shell --pure: let variables for Wayland through We let DISPLAY (X11) through, so we should let the Wayland equivalents through as well. Similarly, we let HOME through, so it should be okay to allow XDG_RUNTIME_DIR (which is needed for connecting to Wayland with WAYLAND_DISPLAY) through as well. Otherwise graphical applications will either fall back to X11 (if they support it), or just not work (if they don't). --- src/nix-build/nix-build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 73d93480e..75576ef8a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -105,7 +105,8 @@ static void main_nix_build(int argc, char * * argv) // List of environment variables kept for --pure std::set keepVars{ - "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", + "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", + "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; From 0768c08d999593fc23675b17c6f0480a35206c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Fro=C5=82ow?= Date: Mon, 22 Nov 2021 13:35:35 +0100 Subject: [PATCH 73/85] Typo: change to normal singlequote --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5bd4e5545..09007a22c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2710,9 +2710,9 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. )", .fun = prim_foldlStrict, From db2e4489a54c1c4bf5bd8fd2deb8b3fd02a712f1 Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Mon, 22 Nov 2021 13:57:56 +0100 Subject: [PATCH 74/85] Unify #if linux --- src/libstore/gc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 29d45e067..7a414da6b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -324,7 +324,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -#if defined(__linux__) +#if __linux__ static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -416,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #endif -#if defined(__linux__) +#if __linux__ readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); From f68699963c51d343e95969397f436af73ba4b1b4 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Mon, 22 Nov 2021 17:57:30 +0100 Subject: [PATCH 75/85] flake: Do not use aliases gmock is not available with `nixpkgs.config.allowAliases = false`. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index fd5e18429..01afc23c3 100644 --- a/flake.nix +++ b/flake.nix @@ -91,7 +91,7 @@ libarchive boost lowdown-nix - gmock + gtest ] ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium From 08b1ac3e389cda2ba0947e25ea7b09ba2df4813f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 22 Nov 2021 14:42:31 -0600 Subject: [PATCH 76/85] Set new rosetta 2 path in sandbox see: https://github.com/NixOS/nix/pull/5388 and https://github.com/NixOS/nix/pull/5251 --- src/libstore/sandbox-defaults.sb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 41893e6dd..56b35c3fe 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -100,4 +100,5 @@ ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. (allow file-read* - (subpath "/Library/Apple/usr/libexec/oah")) + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah")) From 5be8fbd7403b0b2df268ca79a3dc93cae8f55758 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 22:01:42 +0000 Subject: [PATCH 77/85] Bump cachix/install-nix-action from 15 to 16 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 15 to 16. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v15...v16) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be4fcd8d1..1b655e27d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v15 + - uses: cachix/install-nix-action@v16 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 if: needs.check_cachix.outputs.secret == 'true' @@ -50,7 +50,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v15 + - uses: cachix/install-nix-action@v16 - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' @@ -69,7 +69,7 @@ jobs: steps: - uses: actions/checkout@v2.4.0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v15 + - uses: cachix/install-nix-action@v16 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" From cd72a8c346ca752441327cce4a2e94a2b38055bd Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 23 Nov 2021 11:31:05 +0100 Subject: [PATCH 78/85] Make docker image downloadable in Hydra UI --- flake.nix | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 01afc23c3..8a90de5e2 100644 --- a/flake.nix +++ b/flake.nix @@ -407,10 +407,18 @@ # docker image with Nix inside dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: - import ./docker.nix { + let pkgs = nixpkgsFor.${system}; - tag = version; - }); + image = import ./docker.nix { inherit pkgs; tag = version; }; + in pkgs.runCommand "docker-image-tarball-${version}" + { meta.description = "Docker image with Nix for ${system}"; + } + '' + mkdir -p $out/nix-support + image=$out/image.tar.gz + cp ${image} $image + echo "file binary-dist $image" >> $out/nix-support/hydra-build-products + ''); # Line coverage analysis. coverage = From 861404a87b2c551ebc189f91ae00d21c1287346a Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 23 Nov 2021 12:55:49 +0000 Subject: [PATCH 79/85] Add missing Nix::Store import to fix nix-serve StoreDir. --- perl/lib/Nix/Config.pm.in | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index f7c6f2484..508a15e15 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -1,6 +1,7 @@ package Nix::Config; use MIME::Base64; +use Nix::Store; $version = "@PACKAGE_VERSION@"; From b26cb0c9ac406e03ee54aa07d36350feae8c9bfc Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Tue, 23 Nov 2021 16:15:34 +0100 Subject: [PATCH 80/85] Fix use after free in content-address.cc Inspired by https://github.com/NixOS/nix/pull/5599 --- src/libstore/content-address.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 974d1c471..cf32ccdc4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { - std::string_view asPrefix {std::string{caMethod} + ":"}; - return parseContentAddressMethodPrefix(asPrefix); + std::string asPrefix = std::string{caMethod} + ":"; + // parseContentAddressMethodPrefix takes its argument by reference + std::string_view asPrefixView = asPrefix; + return parseContentAddressMethodPrefix(asPrefixView); } std::optional parseContentAddressOpt(std::string_view rawCaOpt) From 52c84c15e5c8b996bcd0c5e13db656769804d05f Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Wed, 24 Nov 2021 09:18:33 +0100 Subject: [PATCH 81/85] Don't copy, to reduce store size --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8a90de5e2..ba2f2a50c 100644 --- a/flake.nix +++ b/flake.nix @@ -416,7 +416,7 @@ '' mkdir -p $out/nix-support image=$out/image.tar.gz - cp ${image} $image + ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''); From e7906ffd0e6e6ec1b6c01d77bf8db446abf6cd51 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Wed, 24 Nov 2021 09:19:29 +0100 Subject: [PATCH 82/85] Add dockerImage to the checks --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index ba2f2a50c..1b98b526e 100644 --- a/flake.nix +++ b/flake.nix @@ -524,6 +524,7 @@ binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; + dockerImage = self.hydraJobs.dockerImage.${system}; }); packages = forAllSystems (system: { From 09471d2680292af48b2788108de56a8da755d661 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 22 Nov 2021 23:56:40 +0100 Subject: [PATCH 83/85] Make lists be comparable Makes lists comparable using lexicographic comparison. Increments builtins.langVersion in order for this change to be detectable --- .../src/expressions/language-operators.md | 8 +++--- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libexpr/primops.cc | 26 +++++++++++++++---- tests/lang/eval-okay-sort.exp | 2 +- tests/lang/eval-okay-sort.nix | 14 +++++++++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index 6ab319039..268b44f4c 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -17,10 +17,10 @@ order of precedence (from strongest to weakest binding). | String Concatenation | *string1* `+` *string2* | left | String concatenation. | 7 | | Not | `!` *e* | none | Boolean negation. | 8 | | Update | *e1* `//` *e2* | right | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes). | 9 | -| Less Than | *e1* `<` *e2*, | none | Arithmetic comparison. | 10 | -| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than | *e1* `>` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic comparison. | 10 | +| Less Than | *e1* `<` *e2*, | none | Arithmetic/lexicographic comparison. | 10 | +| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than | *e1* `>` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | | Equality | *e1* `==` *e2* | none | Equality. | 11 | | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9a29938c2..26c7d2cce 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -3,3 +3,5 @@ * Binary cache stores now have a setting `compression-level`. * `nix develop` now has a flag `--unpack` to run `unpackPhase`. + +* Lists can now be compared lexicographically using the `<` operator. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8dccf6401..69f4737a8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -517,7 +517,11 @@ static RegisterPrimOp primop_isPath({ struct CompareValues { - bool operator () (const Value * v1, const Value * v2) const + EvalState & state; + + CompareValues(EvalState & state) : state(state) { }; + + bool operator () (Value * v1, Value * v2) const { if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; @@ -534,6 +538,17 @@ struct CompareValues return strcmp(v1->string.s, v2->string.s) < 0; case nPath: return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { + return (*this)(v1->listElems()[i], v2->listElems()[i]); + } + } default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } @@ -621,7 +636,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - set doneKeys; + auto cmp = CompareValues(state); + set doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); @@ -2821,7 +2837,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + return CompareValues(state)(a, b); Value * vs[] = {a, b}; Value vBool; @@ -3103,7 +3119,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp; + CompareValues comp{state}; mkBool(v, comp(args[0], args[1])); } @@ -3693,7 +3709,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 5); + mkInt(v, 6); addConstant("__langVersion", v); // Miscellaneous diff --git a/tests/lang/eval-okay-sort.exp b/tests/lang/eval-okay-sort.exp index 148b93516..899119e20 100644 --- a/tests/lang/eval-okay-sort.exp +++ b/tests/lang/eval-okay-sort.exp @@ -1 +1 @@ -[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] ] +[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] [ [ ] [ ] [ 1 ] [ 1 4 ] [ 1 5 ] [ 1 6 ] [ 2 ] [ 2 3 ] [ 3 ] [ 3 ] ] ] diff --git a/tests/lang/eval-okay-sort.nix b/tests/lang/eval-okay-sort.nix index 8299c3a4a..50aa78e40 100644 --- a/tests/lang/eval-okay-sort.nix +++ b/tests/lang/eval-okay-sort.nix @@ -4,5 +4,17 @@ with builtins; (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) (sort (x: y: x.key < y.key) - [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) + [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) + (sort lessThan [ + [ 1 6 ] + [ ] + [ 2 3 ] + [ 3 ] + [ 1 5 ] + [ 2 ] + [ 1 ] + [ ] + [ 1 4 ] + [ 3 ] + ]) ] From 884674a8e20789250929a4cc6be3ad5445836e63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Nov 2021 13:49:07 +0100 Subject: [PATCH 84/85] nix flake check: Fix markdown --- src/nix/flake-check.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index d995d6274..07031c909 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -31,38 +31,38 @@ at the first error. The following flake output attributes must be derivations: * `checks.`*system*`.`*name* -* `defaultPackage.`*system*` -* `devShell.`*system*` -* `devShells.`*system*`.`*name*` -* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `defaultPackage.`*system* +* `devShell.`*system* +* `devShells.`*system*`.`*name* +* `nixosConfigurations.`*name*`.config.system.build.toplevel` * `packages.`*system*`.`*name* The following flake output attributes must be [app definitions](./nix3-run.md): * `apps.`*system*`.`*name* -* `defaultApp.`*system*` +* `defaultApp.`*system* The following flake output attributes must be [template definitions](./nix3-flake-init.md): * `defaultTemplate` -* `templates`.`*name* +* `templates.`*name* The following flake output attributes must be *Nixpkgs overlays*: * `overlay` -* `overlays`.`*name* +* `overlays.`*name* The following flake output attributes must be *NixOS modules*: * `nixosModule` -* `nixosModules`.`*name* +* `nixosModules.`*name* The following flake output attributes must be [bundlers](./nix3-bundle.md): -* `bundlers`.`*name* +* `bundlers.`*name* * `defaultBundler` In addition, the `hydraJobs` output is evaluated in the same way as From 8388d2c7c662e37470240cfde798956fe8e36a6f Mon Sep 17 00:00:00 2001 From: Las Safin Date: Fri, 8 Oct 2021 22:55:08 +0000 Subject: [PATCH 85/85] Make recursive-nix work even when not privileged Before this, `setns` would fail when switching to the mount namespace, since we did not have the privileges to do so when not root. Closes #5360 --- src/libstore/build/local-derivation-goal.cc | 11 +++++++++-- src/libstore/build/local-derivation-goal.hh | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c9a4a31e7..ebcb561c2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -260,6 +260,7 @@ void LocalDerivationGoal::cleanupHookFinally() void LocalDerivationGoal::cleanupPreChildKill() { sandboxMountNamespace = -1; + sandboxUserNamespace = -1; } @@ -906,11 +907,14 @@ void LocalDerivationGoal::startBuilder() "nobody:x:65534:65534:Nobody:/:/noshell\n", sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - /* Save the mount namespace of the child. We have to do this + /* Save the mount- and user namespace of the child. We have to do this *before* the child does a chroot. */ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); + sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxUserNamespace.get() == -1) + throw SysError("getting sandbox user namespace"); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -1423,7 +1427,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); + debug("bind-mounting %s -> %s", source, target); if (pathExists(target)) throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); @@ -1438,6 +1442,9 @@ void LocalDerivationGoal::addDependency(const StorePath & path) child process.*/ Pid child(startProcess([&]() { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); + if (setns(sandboxMountNamespace.get(), 0) == -1) throw SysError("entering sandbox mount namespace"); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 088a57209..bfdf91d89 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -27,9 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; - /* The mount namespace of the builder, used to add additional + /* The mount namespace and user namespace of the builder, used to add additional paths to the sandbox as a result of recursive Nix calls. */ AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; /* On Linux, whether we're doing the build in its own user namespace. */