From 10d33452e289ded93e192c7d99c1da08a52448e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Tue, 17 Apr 2018 09:13:01 +0200 Subject: [PATCH 001/433] =?UTF-8?q?nix-lang=20parser:=C2=A0Add=20mixed=20n?= =?UTF-8?q?ested=20attrs=20tests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of issue #2077 --- tests/lang/parse-fail-mixed-nested-attrs.nix | 4 ++++ tests/lang/parse-okay-mixed-nested-attrs-1.nix | 4 ++++ tests/lang/parse-okay-mixed-nested-attrs-2.nix | 4 ++++ tests/lang/parse-okay-nested-attrs.nix | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 tests/lang/parse-fail-mixed-nested-attrs.nix create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-1.nix create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-2.nix create mode 100644 tests/lang/parse-okay-nested-attrs.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs.nix new file mode 100644 index 000000000..458a7c545 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.z = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 000000000..fd1001c8c --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 000000000..ad066b680 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix new file mode 100644 index 000000000..fd1001c8c --- /dev/null +++ b/tests/lang/parse-okay-nested-attrs.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} From 00584bb0915ee21fb01e9390a901161bdd988197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Wed, 18 Apr 2018 18:39:40 +0200 Subject: [PATCH 002/433] parser: Allow mixed nested and top-level attrpaths Fixes #2077. --- src/libexpr/parser.y | 26 ++++++++++++++++--- ...nix => parse-fail-mixed-nested-attrs1.nix} | 2 +- tests/lang/parse-fail-mixed-nested-attrs2.nix | 4 +++ ...attrs-6.nix => parse-okay-dup-attrs-6.nix} | 0 tests/lang/parse-okay-nested-attrs.nix | 4 --- 5 files changed, 28 insertions(+), 8 deletions(-) rename tests/lang/{parse-fail-mixed-nested-attrs.nix => parse-fail-mixed-nested-attrs1.nix} (100%) create mode 100644 tests/lang/parse-fail-mixed-nested-attrs2.nix rename tests/lang/{parse-fail-dup-attrs-6.nix => parse-okay-dup-attrs-6.nix} (100%) delete mode 100644 tests/lang/parse-okay-nested-attrs.nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index ef11dd609..df4fdf032 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -83,7 +83,9 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { @@ -104,11 +106,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } - if (i->symbol.set()) { + // Expr insertion. + // ========================== + if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + ExprAttrs* ae = dynamic_cast(e); + ExprAttrs* jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto ad: ae->attrs) { + ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs.nix rename to tests/lang/parse-fail-mixed-nested-attrs1.nix index 458a7c545..11e40e66f 100644 --- a/tests/lang/parse-fail-mixed-nested-attrs.nix +++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -1,4 +1,4 @@ { - x = { y = 3; z = 3; }; x.z = 3; + x = { y = 3; z = 3; }; } diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 000000000..17da82e5f --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-6.nix rename to tests/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix deleted file mode 100644 index fd1001c8c..000000000 --- a/tests/lang/parse-okay-nested-attrs.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ - x = { y = 3; z = 3; }; - x.q = 3; -} From b2f3a7411a509d5df5a189066870a0f02b6523a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Tue, 1 May 2018 14:42:34 +0200 Subject: [PATCH 003/433] nix-lang: Add deep nested mixed attrs test case. --- tests/lang/parse-okay-mixed-nested-attrs-3.nix | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 000000000..45a33e480 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} From 41f38fbb4b89dc5df244a07410724beca2e14d96 Mon Sep 17 00:00:00 2001 From: Nathan van Doorn Date: Mon, 16 Jul 2018 10:00:42 +0100 Subject: [PATCH 004/433] nix-channel documentation: don't suggest deprecated function Running `nix-instantiate --eval -E '(import {}).lib.nixpkgsVersion` emits a warning ``` trace: `lib.nixpkgsVersion` is deprecated, use `lib.version` instead! ``` --- doc/manual/command-ref/nix-channel.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a76..f5159165a 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -112,13 +112,13 @@ $ nix-env -iA nixpkgs.hello You can revert channel updates using : -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.526.dbadfad" From 58a85fa4621faaa89286065f5583328783085722 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 8 Aug 2018 21:21:21 +0200 Subject: [PATCH 005/433] mention nix-store --query --roots when a path cannot be deleted --- src/libstore/gc.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b415d5421..65220a9f6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -791,7 +791,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) assertStorePath(i); tryToDelete(state, i); if (state.dead.find(i) == state.dead.end()) - throw Error(format("cannot delete path '%1%' since it is still alive") % i); + throw Error(format( + "cannot delete path '%1%' since it is still alive. " + "To find out why use: " + "nix-store --query --roots" + ) % i); } } else if (options.maxFreed > 0) { From 7e35e914c1aa24957107c666c76f1d834ebae90a Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 14 Dec 2018 20:07:23 +0100 Subject: [PATCH 006/433] fetchGit: allow fetching explicit refs Trying to fetch refs that are not in refs/heads currently fails because it looks for refs/heads/refs/foo instead of refs/foo. eg. builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "refs/pull/1024/head; } --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258..588b0fa4d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -94,7 +94,11 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", "--bare", cacheDir }); } - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); From 82f054d7d5fc8f9de45afa7107557644d1514c98 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 20 Dec 2018 20:05:14 +0100 Subject: [PATCH 007/433] installer: update macOS version check to 10.12.2 Nixpkgs will drop support for <10.12 soon and thus a nix release built using the 19.03 channel will also require a newer version of macOS. --- scripts/install-nix-from-closure.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index ab20774bb..e635dcab3 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then exit 1 fi -# macOS support for 10.10 or higher +# macOS support for 10.12.6 or higher if [ "$(uname -s)" = "Darwin" ]; then - if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi fi From b9567aa8b6f935522d1037377e0e5205a7529fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 13 Feb 2019 11:28:28 +0100 Subject: [PATCH 008/433] install script: don't abort when "nix-channel --update" fails Instead, print a message about what happened and tell the user what can be done (run "[sudo -i] nix-channel --update nixpkgs" again at a later time). This change allows installing Nix when you're offline. Since the multi-user installer is so verbose, the message isn't printed until the end. Fixes issue #2650 ("installation without internet connection"). --- scripts/install-multi-user.sh | 10 +++++++++- scripts/install-nix-from-closure.sh | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..3d2acaac8 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -240,10 +240,16 @@ EOF } trap finish_fail EXIT +channel_update_failed=0 function finish_success { finish_cleanup ok "Alright! We're done!" + if [ "x$channel_update_failed" = x1 ]; then + echo "" + echo "But fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." + fi cat < Date: Sun, 24 Mar 2019 09:39:48 +0000 Subject: [PATCH 009/433] repl: Restore CTRL-C behaviour Install signal handler during `readline` to handle SIGINT to abort partially typed expressions. --- src/nix/repl.cc | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 227affc60..d8f812149 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -192,6 +192,14 @@ static int listPossibleCallback(char *s, char ***avp) { return ac; } +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} void NixRepl::mainLoop(const std::vector & files) { @@ -251,8 +259,40 @@ void NixRepl::mainLoop(const std::vector & files) bool NixRepl::getLine(string & input, const std::string &prompt) { + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + if (!s) return false; input += s; From d854e7dfd6d512c8ed687b61d7aae3358eda71cd Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Tue, 26 Mar 2019 21:08:22 -0400 Subject: [PATCH 010/433] install-multi-user: Detect and fail lack of systemd separately Otherwise, the user is shown: ``` Sorry, I don't know what to do on Linux ``` Which is... not exactly right. --- scripts/install-multi-user.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 4b65783a2..80d1c73fb 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -753,9 +753,13 @@ main() { if [ "$(uname -s)" = "Darwin" ]; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi else failure "Sorry, I don't know what to do on $(uname)" fi From dbe4c043d75f90f30932d326d2c80fd030b1f06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 27 Mar 2019 16:18:53 +0100 Subject: [PATCH 011/433] install-multi-user: reduce max-jobs from 32 to 1 Having max-jobs = 32 ($NIX_USER_COUNT is hardcoded to that value) may severely overload the machine. The nix.conf(5) manual page says max-jobs defaults to 1, so let's use that value. NOTE: Both max-jobs and cores are now being set to their default value, so they can be removed alltogether. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 4b65783a2..6a76761cc 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -742,7 +742,7 @@ place_nix_configuration() { cat < "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME -max-jobs = $NIX_USER_COUNT +max-jobs = 1 cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ From 07d9981f34c8423e74a8dc1b4f978fe58b421aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 27 Mar 2019 16:24:58 +0100 Subject: [PATCH 012/433] install-multi-user: remove unneeded settings from nix.conf Hardcoding the "max-jobs" and "cores" settings in nix.conf at install time, to the same value as Nix' built-in default, makes little sense to me. --- scripts/install-multi-user.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6a76761cc..1e1db21c3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -741,9 +741,6 @@ setup_default_profile() { place_nix_configuration() { cat < "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME - -max-jobs = 1 -cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf From caa76c369a6d77eb3ccaa14bd8144581c8cd50a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sun, 31 Mar 2019 03:47:43 +0200 Subject: [PATCH 013/433] docs: Mention `--max-jobs 0` to build remotely only --- doc/manual/advanced-topics/distributed-builds.xml | 3 +++ doc/manual/command-ref/opt-common.xml | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index ce2e077ed..9ac4a92cd 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -184,4 +184,7 @@ to be included. (This is the default.) the option builders-use-substitutes in your local nix.conf. +To build only on remote builders and disable building on the local machine, +you can use the option . + diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 4c572e129..b8a2f260e 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -107,14 +107,22 @@ / number - Sets the maximum number of build jobs that Nix will + + + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify auto to use the number of CPUs in the system. The default is specified by the max-jobs configuration setting, which itself defaults to 1. A higher value is useful on SMP systems or to - exploit I/O latency. + exploit I/O latency. + + Setting it to 0 disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders. + + From 288f93cec0efeba5a767c09e4d58203c67704a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sun, 14 Apr 2019 03:18:33 +0200 Subject: [PATCH 014/433] manual: "Nix Package collection" -> "Nixpkgs package collection". Makes difference between Nix and Nixpkgs clearer to avoid some common confusion this sentence on IRC. Also disambiguate an "it" reference. --- doc/manual/packages/basic-package-mgmt.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml index e8d1419da..0f21297f3 100644 --- a/doc/manual/packages/basic-package-mgmt.xml +++ b/doc/manual/packages/basic-package-mgmt.xml @@ -24,11 +24,11 @@ symlinks to the files of the active applications. Components are installed from a set of Nix expressions that tell Nix how to build those packages, including, if necessary, their dependencies. There is a collection of -Nix expressions called the Nix Package collection that contains +Nix expressions called the Nixpkgs package collection that contains packages ranging from basic development stuff such as GCC and Glibc, to end-user applications like Mozilla Firefox. (Nix is however not -tied to the Nix Package collection; you could write your own Nix -expressions based on it, or completely new ones.) +tied to the Nixpkgs package collection; you could write your own Nix +expressions based on Nixpkgs, or completely new ones.) You can manually download the latest version of Nixpkgs from . However, From 5112a33fb17f792ceb6d641738277cbbe6a58bfc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 19:17:17 +0200 Subject: [PATCH 015/433] Fix release script --- maintainers/upload-release.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 8432c9596..1cdf5ed16 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -67,10 +67,10 @@ sub downloadFile { } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt\n"; + print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } From b614e0e53da1f825e198996f88935a1d1a89725f Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Wed, 17 Apr 2019 21:42:41 +0200 Subject: [PATCH 016/433] build: only skip hash rewriting for sandboxing on linux The sandbox on darwin, and possibly other future platforms, doesn't have enough isolation to redirect outputs without hash rewriting. --- src/libstore/build.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0c743b..dbadfacd5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -803,6 +803,9 @@ private: /* Whether we're currently doing a chroot build. */ bool useChroot = false; + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite; + Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -994,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , wantedOutputs(wantedOutputs) , buildMode(buildMode) { +#if __linux__ + needsHashRewrite = !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + needsHashRewrite = true; +#endif + state = &DerivationGoal::getDerivation; name = (format("building of '%1%'") % drvPath).str(); trace("created"); @@ -2073,7 +2083,7 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -2873,6 +2883,10 @@ void DerivationGoal::runChild() for (auto & i : missingPaths) { sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -3051,7 +3065,9 @@ void DerivationGoal::registerOutputs() throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { + } + + if (needsHashRewrite) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() From cbc7d9a4124343c7ba68b695b5e9b3c8188fb267 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Tue, 30 Apr 2019 22:43:24 -0400 Subject: [PATCH 017/433] findRootsNoTemp: fixes comment about findRuntimeRoots The NIX_ROOT_FINDER environment variable was removed in 3c46fe62b833a4e66845665edc99555022d3d98c when porting from perl to C. --- src/libstore/gc.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d8a5da0d4..26e2b0dca 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -326,10 +326,9 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor) findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ findRuntimeRoots(roots, censor); } From f9a2ea44867cd1dbb408bca4df0ced806137b7f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 May 2019 10:44:32 +0200 Subject: [PATCH 018/433] Fix "Bad system call" running i686-linux binaries on x86_64-linux To determine which seccomp filters to install, we were incorrectly using settings.thisSystem, which doesn't denote the actual system when --system is used. Fixes #2791. --- src/libstore/build.cc | 8 ++++---- src/libutil/util.cc | 3 +++ src/libutil/util.hh | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dbadfacd5..91eb97dfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2510,17 +2510,17 @@ void setupSeccomp() seccomp_release(ctx); }); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) throw SysError("unable to add X32 seccomp architecture"); - if (settings.thisSystem == "aarch64-linux" && + if (nativeSystem == "aarch64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes."); + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e3dcd246c..a71705665 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -38,6 +38,9 @@ extern char * * environ; namespace nix { +const std::string nativeSystem = SYSTEM; + + BaseError & BaseError::addPrefix(const FormatOrString & fs) { prefix_ = fs.s + prefix_; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..54936a5cb 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -30,6 +30,10 @@ struct Sink; struct Source; +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); From 3f192ac80ca421888c668896b63485486e1397ae Mon Sep 17 00:00:00 2001 From: Daniel Schaefer Date: Fri, 3 May 2019 14:30:29 +0200 Subject: [PATCH 019/433] Add builtins.hashFile For text files it is possible to do it like so: `builtins.hashString "sha256" (builtins.readFile /tmp/a)` but that doesn't work for binary files. With builtins.hashFile any kind of file can be conveniently hashed. --- doc/manual/expressions/builtins.xml | 13 +++++++++++++ src/libexpr/primops.cc | 15 +++++++++++++++ tests/lang/binary-data | Bin 0 -> 1024 bytes tests/lang/eval-fail-hashfile-missing.nix | 5 +++++ tests/lang/eval-okay-hash.exp | 1 - tests/lang/eval-okay-hashfile.exp | 1 + tests/lang/eval-okay-hashfile.nix | 4 ++++ tests/lang/eval-okay-hashstring.exp | 1 + ...al-okay-hash.nix => eval-okay-hashstring.nix} | 0 9 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/lang/binary-data create mode 100644 tests/lang/eval-fail-hashfile-missing.nix create mode 100644 tests/lang/eval-okay-hashfile.exp create mode 100644 tests/lang/eval-okay-hashfile.nix create mode 100644 tests/lang/eval-okay-hashstring.exp rename tests/lang/{eval-okay-hash.nix => eval-okay-hashstring.nix} (100%) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 0fb5261b3..a87639a07 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -705,6 +705,19 @@ builtins.genList (x: x * x) 5 + + builtins.hashFile + type p + + Return a base-16 representation of the + cryptographic hash of the file at path p. The + hash algorithm specified by type must + be one of "md5", "sha1", + "sha256" or "sha512". + + + + builtins.head list diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 39073725e..06f577f36 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -923,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2202,6 +2216,7 @@ void EvalState::createBaseEnv() addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); diff --git a/tests/lang/binary-data b/tests/lang/binary-data new file mode 100644 index 0000000000000000000000000000000000000000..06d7405020018ddf3cacee90fd4af10487da3d20 GIT binary patch literal 1024 RcmZP=1*0J_8UiCW1ONm800961 literal 0 HcmV?d00001 diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix new file mode 100644 index 000000000..42fb1ec7e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -0,0 +1,5 @@ +let + paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) + diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp index d720a082d..e69de29bb 100644 --- a/tests/lang/eval-okay-hash.exp +++ b/tests/lang/eval-okay-hash.exp @@ -1 +0,0 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/lang/eval-okay-hashfile.exp new file mode 100644 index 000000000..ff1e8293e --- /dev/null +++ b/tests/lang/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/lang/eval-okay-hashfile.nix new file mode 100644 index 000000000..aff5a1856 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/lang/eval-okay-hashstring.exp new file mode 100644 index 000000000..d720a082d --- /dev/null +++ b/tests/lang/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hash.nix rename to tests/lang/eval-okay-hashstring.nix From a83486187616fd2de877ce72f5fd419385a2c541 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Tue, 7 May 2019 17:32:01 -0500 Subject: [PATCH 020/433] fix hashfile test that wasn't failing due to eval laziness See: https://github.com/NixOS/nix/commit/7becb1bf1c2ec1544a5374580a97b36273506baf#r33450554 --- tests/lang/eval-fail-hashfile-missing.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix index 42fb1ec7e..ce098b823 100644 --- a/tests/lang/eval-fail-hashfile-missing.nix +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -1,5 +1,5 @@ let paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; in - builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) + toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) From 7c6391ddc730519a632cc0ee526c94a04812d871 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 14:18:57 +0200 Subject: [PATCH 021/433] nix-shell: Don't absolutize '-p' expressions This prevents spurious syscalls like 25011 lstat("/home/eelco/with import { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ (hello) ]; } \"\"", 0x7ffe9c67f580) = -1 ENOENT (No such file or directory) --- src/nix-build/nix-build.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 618895d38..c6a4d4166 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -274,19 +274,21 @@ static void _main(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : left) { - auto absolute = i; - try { - absolute = canonPath(absPath(i), true); - } catch (Error e) {}; if (fromArgs) exprs.push_back(state->parseExprFromString(i, absPath("."))); - else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error e) {}; + if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + } } /* Evaluate them into derivations. */ From 6ade7ec022c836b7d1f9bd06be45e2c07835ec8c Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 6 May 2019 22:23:15 +0200 Subject: [PATCH 022/433] progress-bar: hide expected if expected is 0 (unknown) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes, "expected" can be "0", but in fact means "unknown". This is for example the case when downloading a file while the http server doesn't send the `Content-Length` header, like when running `nix build` pointing to a nixpkgs checkout streamed from GitHub: ⇒ nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello [1.8/0.0 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz' In that case, don't show that weird progress bar, but only the (slowly increasing) downloaded size ("done"). ⇒ nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello [1.8 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz' This commit also updates fmt calls with three numbers (when something is currently 'running' too) - I'm not sure if this can be provoked, but showing "0" as expected doesn't make any sense, as we're obviously doing more than nothing. --- src/nix/progress-bar.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3..8da72bc94 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -333,11 +333,18 @@ public: if (running || done || expected || failed) { if (running) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); else if (expected != done) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); else s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); s = fmt(itemFmt, s); From ff6867ab94cbe9ddcb4ba18d68a4a2dcb79b865d Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 May 2019 15:53:40 +0200 Subject: [PATCH 023/433] build: move needsHashRewrite initialization to startBuilder The value of useChroot is not set yet in the constructor, resulting in hash rewriting being enabled in certain cases where it should not be. Fixes #2801 --- src/libstore/build.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 91eb97dfb..30825add4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -997,13 +997,6 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , wantedOutputs(wantedOutputs) , buildMode(buildMode) { -#if __linux__ - needsHashRewrite = !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - needsHashRewrite = true; -#endif - state = &DerivationGoal::getDerivation; name = (format("building of '%1%'") % drvPath).str(); trace("created"); @@ -1852,6 +1845,13 @@ void DerivationGoal::startBuilder() #endif } +#if __linux__ + needsHashRewrite = !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + needsHashRewrite = true; +#endif + /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ if (settings.buildUsersGroup != "" && getuid() == 0) { From d75bdb5793e5ebf9e480f5a0012d141347725801 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 May 2019 16:46:21 +0200 Subject: [PATCH 024/433] build: add test for sandboxed --check --- tests/linux-sandbox.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index acfd46c54..52967d07d 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -25,3 +25,6 @@ nix path-info -r $outPath | grep input-2 nix ls-store -R -l $outPath | grep foobar nix cat-store $outPath/foobar | grep FOOBAR + +# Test --check without hash rewriting. +nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store From c78686e411e0a14cff51836fe6c35d7584171df3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 May 2019 16:39:31 -0400 Subject: [PATCH 025/433] build: run diff-hook under --check and document diff-hook --- .../advanced-topics/advanced-topics.xml | 1 + doc/manual/advanced-topics/diff-hook.xml | 207 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 81 ++++++- src/libstore/build.cc | 30 ++- 4 files changed, 303 insertions(+), 16 deletions(-) create mode 100644 doc/manual/advanced-topics/diff-hook.xml diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index b710f9f2b..c304367aa 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,5 +7,6 @@ Advanced Topics + diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 000000000..d2613f6df --- /dev/null +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,207 @@ + + +Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option> + +Check build reproducibility by running builds multiple times +and comparing their results. + +Specify a program with Nix's to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same. + +For purposes of demonstration, we'll use the following Nix file, +deterministic.nix for testing: + + +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} + + +Additionally, nix.conf contains: + + +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true + + +where /etc/nix/my-diff-hook is an executable +file containing: + + +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2" + + + + The diff hook can be run as root. Take care to run as little + as possible as root, for this example we use runuser + to drop privileges. + + + + +
+ + Spot-Checking Build Determinism + + + + Verify a path which already exists in the Nix store by passing + to the build command. + + + If the build passes and is deterministic, Nix will exit with a + status code of 0: + + +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + + + If the build is not deterministic, Nix will exit with a status + code of 1: + + +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs + + +In the Nix daemon's log, we will now see: + +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 + + + + Using with + will cause Nix to keep the second build's output in a special, + .check path: + + +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' + + + In particular, notice the + /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check + output. Nix has copied the build results to that directory where you + can examine it. + + + <literal>.check</literal> paths are not registered store paths + + Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection. + + The path is guaranteed to be alive for the duration of + 's execution, but may be deleted + any time after. + + If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist. + + + + is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible + + + Run the build without , and then try with + again. + +
+ +
+ + Automatic and Optionally Enforced Determinism Verification + + + + Automatically verify every build at build time by executing the + build multiple times. + + + + Setting and + in your + nix.conf permits the automated verification + of every build Nix performs. + + + + The following configuration will run each build three times, and + will require the build to be deterministic: + + +enforce-determinism = true +repeat = 2 + + + + + Setting to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + +enforce-determinism = false +repeat = 1 + + + + + An example output of this configuration: + +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable + + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index f0da1f612..a1a5d6e12 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -1,7 +1,9 @@ + + xml:id="sec-conf-file" + version="5"> nix.conf @@ -240,6 +242,64 @@ false. + diff-hook + + + Absolute path to an executable capable of diffing build results. + The hook executes if is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + + + + + The root user executes the diff hook in a daemonised + installation. See for + information on using the diff hook safely. + + + + The diff hook program receives three parameters: + + + + + A path to the previous build's results + + + + + + A path to the current build's results + + + + + + The path to the build's derivation + + + + + The diff hook should not print data to stderr or stdout, as + output is not displayed to the user. However, if information is + printed, it will be printed in the nix-daemon + log. + + When using the Nix daemon, diff-hook must + be set in the nix.conf configuration file, and + cannot be passed at the command line. + + + + + + enforce-determinism + + See . + + extra-sandbox-paths @@ -595,9 +655,9 @@ password my-password they are deterministic. The default value is 0. If the value is non-zero, every build is repeated the specified number of times. If the contents of any of the runs differs from the - previous ones, the build is rejected and the resulting store paths - are not registered as “valid” in Nix’s database. - + previous ones and is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database. require-sigs @@ -628,6 +688,19 @@ password my-password + run-diff-hook + + + If true, enable the execution of . + + + + When using the Nix daemon, run-diff-hook must + be set in the nix.conf configuration file, + and cannot be passed at the command line. + + + sandbox diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 91eb97dfb..026828c53 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,6 +461,19 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } +void handleDiffHook(Path tryA, Path tryB, Path drvPath) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } +} ////////////////////////////////////////////////////////////////////// @@ -3039,8 +3052,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.runDiffHook; - bool keepPreviousRound = settings.keepFailed || runDiffHook; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -3185,11 +3197,14 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + handleDiffHook(path, actualPath, drvPath); + if (settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else @@ -3254,16 +3269,7 @@ void DerivationGoal::registerOutputs() ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); - auto diffHook = settings.diffHook; - if (prevExists && diffHook != "" && runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {prev, i->second.path}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - } + handleDiffHook(prev, i->second.path, drvPath); if (settings.enforceDeterminism) throw NotDeterministic(msg); From 6df61db0600ca73ccd51e3e5bec5312a04e99da1 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 May 2019 20:59:39 -0400 Subject: [PATCH 026/433] diff hook: execute as the build user, and pass the temp dir --- doc/manual/advanced-topics/diff-hook.xml | 12 +++---- doc/manual/command-ref/conf-file.xml | 20 ++++++++---- src/libstore/build.cc | 41 +++++++++++++++++------- src/libutil/util.cc | 4 +-- src/libutil/util.hh | 2 ++ 5 files changed, 51 insertions(+), 28 deletions(-) diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml index d2613f6df..fb4bf819f 100644 --- a/doc/manual/advanced-topics/diff-hook.xml +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -46,17 +46,15 @@ file containing: #!/bin/sh exec >&2 echo "For derivation $3:" -/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2" +/run/current-system/sw/bin/diff -r "$1" "$2" - - The diff hook can be run as root. Take care to run as little - as possible as root, for this example we use runuser - to drop privileges. - - +The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built. +
Spot-Checking Build Determinism diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index a1a5d6e12..c5f90481b 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -252,13 +252,11 @@ false</literal>.</para> same. </para> - <warning> - <para> - The root user executes the diff hook in a daemonised - installation. See <xref linkend="chap-diff-hook" /> for - information on using the diff hook safely. - </para> - </warning> + <para> + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + </para> <para>The diff hook program receives three parameters:</para> @@ -280,6 +278,14 @@ false</literal>.</para> The path to the build's derivation </para> </listitem> + + <listitem> + <para> + The path to the build's scratch directory. This directory + will exist only if the build was run with + <option>--keep-failed</option>. + </para> + </listitem> </orderedlist> <para>The diff hook should not print data to stderr or stdout, as diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 026828c53..f38d2eaa0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,17 +461,26 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(Path tryA, Path tryB, Path drvPath) +void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } + auto wrapper = [&]() { + if (setgid(gid) == -1) + throw SysError("setgid failed"); + if (setuid(uid) == -1) + throw SysError("setuid failed"); + + try { + auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + }; + + doFork(allowVfork, wrapper); } } @@ -3197,14 +3206,18 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { - handleDiffHook(path, actualPath, drvPath); - - if (settings.keepFailed) { + if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + handleDiffHook( + !buildUser && !drv->isBuiltin(), + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, drvPath, tmpDir); + throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else @@ -3269,7 +3282,11 @@ void DerivationGoal::registerOutputs() ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); - handleDiffHook(prev, i->second.path, drvPath); + handleDiffHook( + !buildUser && !drv->isBuiltin(), + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, i->second.path, drvPath, tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a71705665..0f4d3d92b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -914,8 +914,8 @@ void killUser(uid_t uid) /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function<void()> fun) +pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +pid_t doFork(bool allowVfork, std::function<void()> fun) { #ifdef __linux__ pid_t pid = allowVfork ? vfork() : fork(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 54936a5cb..824a35b98 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -265,6 +265,8 @@ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); +pid_t doFork(bool allowVfork, std::function<void()> fun); + struct RunOptions { Path program; From dde8eeb39ae9fb73011462c74e5fa6405e432147 Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sat, 11 May 2019 15:57:38 -0400 Subject: [PATCH 027/433] chdir, setgroups --- src/libstore/build.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f38d2eaa0..8397cd0d1 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -466,8 +466,12 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { auto wrapper = [&]() { + if (chdir("/") == -1) + throw SysError("chdir / failed"); if (setgid(gid) == -1) throw SysError("setgid failed"); + if (setgroups(0, 0) == -1) + throw SysError("setgroups failed"); if (setuid(uid) == -1) throw SysError("setuid failed"); From b4a05edbfe49f87555fd284dfb0d6c56ed43217d Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sat, 11 May 2019 16:35:53 -0400 Subject: [PATCH 028/433] runProgram: support gid, uid, chdir --- src/libstore/build.cc | 32 ++++++++++++++------------------ src/libutil/util.cc | 15 +++++++++++++-- src/libutil/util.hh | 5 +++-- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8397cd0d1..8902e22bd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -465,26 +465,22 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { - auto wrapper = [&]() { - if (chdir("/") == -1) - throw SysError("chdir / failed"); - if (setgid(gid) == -1) - throw SysError("setgid failed"); - if (setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (setuid(uid) == -1) - throw SysError("setuid failed"); + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; - try { - auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - }; + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); - doFork(allowVfork, wrapper); + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0f4d3d92b..55b9144f3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -16,6 +16,7 @@ #include <future> #include <fcntl.h> +#include <grp.h> #include <limits.h> #include <pwd.h> #include <sys/ioctl.h> @@ -914,8 +915,8 @@ void killUser(uid_t uid) /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); -pid_t doFork(bool allowVfork, std::function<void()> fun) +static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function<void()> fun) { #ifdef __linux__ pid_t pid = allowVfork ? vfork() : fork(); @@ -1025,6 +1026,16 @@ void runProgram2(const RunOptions & options) if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); + //if (options.chdir && chdir((*options.chdir).c_str()) == -1) + // throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + Strings args_(options.args); args_.push_front(options.program); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 824a35b98..7c57d0afa 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -265,10 +265,11 @@ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); -pid_t doFork(bool allowVfork, std::function<void()> fun); - struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; Path program; bool searchPath = true; Strings args; From a5efe617862484ab7dd234a495d315e7b08aa519 Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 13:23:30 -0400 Subject: [PATCH 029/433] Clarify where output from the diff hook goes. --- doc/manual/command-ref/conf-file.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c5f90481b..24fbf28cf 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -288,10 +288,11 @@ false</literal>.</para> </listitem> </orderedlist> - <para>The diff hook should not print data to stderr or stdout, as - output is not displayed to the user. However, if information is - printed, it will be printed in the <command>nix-daemon</command> - log.</para> + <para> + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + </para> <para>When using the Nix daemon, <literal>diff-hook</literal> must be set in the <filename>nix.conf</filename> configuration file, and From 73b797c207e1c7a0fd9059d2cf1e3479502f8f1b Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 13:44:22 -0400 Subject: [PATCH 030/433] handleDiffHook: stop passing allowVfork --- src/libstore/build.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8902e22bd..b07461013 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,7 +461,7 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { @@ -3213,7 +3213,6 @@ void DerivationGoal::registerOutputs() throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); handleDiffHook( - !buildUser && !drv->isBuiltin(), buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); @@ -3283,7 +3282,6 @@ void DerivationGoal::registerOutputs() : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); handleDiffHook( - !buildUser && !drv->isBuiltin(), buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), prev, i->second.path, drvPath, tmpDir); From ce02fc74b2db35e45906865c8a3ce2e98871eeb8 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan <daiderd@gmail.com> Date: Sun, 12 May 2019 22:47:41 +0200 Subject: [PATCH 031/433] build: make needsHashRewrite a method --- src/libstore/build.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 30825add4..79dcdddbe 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -803,9 +803,6 @@ private: /* Whether we're currently doing a chroot build. */ bool useChroot = false; - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite; - Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -885,6 +882,9 @@ public: Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + void timedOut() override; string key() override @@ -1037,6 +1037,17 @@ DerivationGoal::~DerivationGoal() } +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + void DerivationGoal::killChild() { if (pid != -1) { @@ -1845,13 +1856,6 @@ void DerivationGoal::startBuilder() #endif } -#if __linux__ - needsHashRewrite = !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - needsHashRewrite = true; -#endif - /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ if (settings.buildUsersGroup != "" && getuid() == 0) { @@ -2083,7 +2087,7 @@ void DerivationGoal::startBuilder() #endif } - if (needsHashRewrite) { + if (needsHashRewrite()) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -3067,7 +3071,7 @@ void DerivationGoal::registerOutputs() if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } - if (needsHashRewrite) { + if (needsHashRewrite()) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() From f1b8e9efe77014655f059b44afa05c38990dc4aa Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 17:03:01 -0400 Subject: [PATCH 032/433] runProgram: Uncomment chdir support --- src/libutil/util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 55b9144f3..17aee2d5c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1026,8 +1026,8 @@ void runProgram2(const RunOptions & options) if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); - //if (options.chdir && chdir((*options.chdir).c_str()) == -1) - // throw SysError("chdir failed"); + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); if (options.gid && setgid(*options.gid) == -1) throw SysError("setgid failed"); /* Drop all other groups if we're setgid. */ From 3fd5425f948afa5122ff6b0aad60a6b961b57161 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 13:13:14 +0200 Subject: [PATCH 033/433] Fix shellcheck error https://hydra.nixos.org/build/93359951 --- scripts/install-nix-from-closure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 4dd249923..fc999d336 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -26,7 +26,7 @@ fi if [ "$(uname -s)" = "Darwin" ]; then macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) - if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi From b6eb8a2d7e2ea8b083fdac15f537679ffe633183 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 14:30:09 +0200 Subject: [PATCH 034/433] nix-profile: Add all channels to $NIX_PATH Fixes #2709. --- scripts/nix-profile.sh.in | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index db03e16ba..f3cfa157c 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -51,10 +51,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then unset __nix_defexpr fi - # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that - # <nixpkgs> paths work when the user has fetched the Nixpkgs - # channel. - export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs> + # paths work when the user has fetched the Nixpkgs channel. + export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix From 66b8a62101cb1dfe2e368346cf99efd32e9328ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 17:33:56 +0200 Subject: [PATCH 035/433] nix: Add --print-build-logs flag This causes 'nix' to print build log output to stderr rather than showing the last log line in the progress bar. Log lines are prefixed by the name of the derivation (minus the version string), e.g. binutils> make[1]: Leaving directory '/build/binutils-2.31.1' binutils-wrapper> unpacking sources binutils-wrapper> patching sources ... binutils-wrapper> Using dynamic linker: '/nix/store/kr51dlsj9v5cr4n8700jliyz8v5b2q7q-bootstrap-stage0-glibc/lib/ld-linux-x86-64.so.2' bootstrap-stage2-gcc-wrapper> unpacking sources ... linux-headers> unpacking sources linux-headers> unpacking source archive /nix/store/8javli69jhj3bkql2c35gsj5vl91p382-linux-4.19.16.tar.xz --- src/libutil/util.hh | 1 + src/nix/main.cc | 10 ++++++++-- src/nix/progress-bar.cc | 39 +++++++++++++++++++++++++++++---------- src/nix/progress-bar.hh | 2 +- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..23360de2e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,6 +401,7 @@ void ignoreException(); /* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" #define ANSI_RED "\e[31;1m" #define ANSI_GREEN "\e[32;1m" #define ANSI_BLUE "\e[34;1m" diff --git a/src/nix/main.cc b/src/nix/main.cc index 64c1dc357..4f87ad72b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -20,6 +20,8 @@ std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { mkFlag() @@ -41,6 +43,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs throw Exit(); }); + mkFlag() + .longName("print-build-logs") + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + mkFlag() .longName("version") .description("show version information") @@ -98,8 +105,7 @@ void mainWrapped(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) - startProgressBar(); + startProgressBar(args.printBuildLogs); args.command->prepare(); args.command->run(); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3..304f918cc 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "sync.hh" #include "store-api.hh" +#include "names.hh" #include <atomic> #include <map> @@ -38,6 +39,7 @@ private: std::map<ActivityType, uint64_t> expectedByType; bool visible = true; ActivityId parent; + std::optional<std::string> name; }; struct ActivitiesByType @@ -68,10 +70,16 @@ private: std::condition_variable quitCV, updateCV; + bool printBuildLogs; + bool isTTY; + public: - ProgressBar() + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) { + state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { @@ -109,8 +117,14 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - draw(state); + if (state.active) { + writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -141,6 +155,7 @@ public: auto nrRounds = getI(fields, 3); if (nrRounds != 1) i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; } if (type == actSubstitute) { @@ -217,11 +232,15 @@ public: auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(); + if (printBuildLogs) { + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(); + } } } @@ -395,9 +414,9 @@ public: } }; -void startProgressBar() +void startProgressBar(bool printBuildLogs) { - logger = new ProgressBar(); + logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO)); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index af8eda5a8..4d61175c2 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -4,7 +4,7 @@ namespace nix { -void startProgressBar(); +void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); From 7c20ee448fa924d898bcebf84bd0a7caf368a656 Mon Sep 17 00:00:00 2001 From: Matthew Bauer <mjbauer95@gmail.com> Date: Wed, 15 May 2019 22:04:39 -0400 Subject: [PATCH 036/433] Sync NIX_PROFILES between single-user and multi-user modes When we are in single user mode, we still want to have access to profiles. This way things in Nixpkgs that rely on them getting set accurately are done in both cases. The point where I hit this is with using aspell which looks in NIX_PROFILES: https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/libraries/aspell/default.nix Before this patch, NIX_PROFILES was never set in single user mode! This corrects that. --- scripts/nix-profile.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index f3cfa157c..85f1d6e5d 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -57,7 +57,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -79,5 +79,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES + unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR fi From 92f461e4f46b595e3712b41c30b8403905dd8673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer <mjbauer95@gmail.com> Date: Wed, 15 May 2019 22:24:24 -0400 Subject: [PATCH 037/433] =?UTF-8?q?Don=E2=80=99t=20set=20NIX=5FREMOTE=3Dda?= =?UTF-8?q?emon=20in=20daemon=20profile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is now autodetected. There is no need to put it in the profile. --- scripts/nix-profile-daemon.sh.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 6940969cc..23da5e855 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -2,12 +2,6 @@ if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -# Set up secure multi-user builds: non-root users build through the -# Nix daemon. -if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then - export NIX_REMOTE=daemon -fi - export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" From c0559a1d6092b94c7ce46a065d72fc74e4db51dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <v@cunat.cz> Date: Fri, 17 May 2019 09:50:42 +0200 Subject: [PATCH 038/433] docs: describe $IN_NIX_SHELL values (#2796) See commit 1bffd83e1a9 --- doc/manual/command-ref/env-common.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index c532ffdde..6a3aaae71 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -14,7 +14,8 @@ <varlistentry><term><envar>IN_NIX_SHELL</envar></term> <listitem><para>Indicator that tells if the current environment was set up by - <command>nix-shell</command>.</para></listitem> + <command>nix-shell</command>. Since Nix 2.0 the values are + <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem> </varlistentry> From 14c877b4ab097667734b050a79b45658dcf2695d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 19 Apr 2019 14:41:59 +0200 Subject: [PATCH 039/433] fetchGit -> fetchTarball (cherry picked from commit cbfdea685764bf66443a999e672656c54289b8c9) --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index ab13451ff..78b39108f 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index 73e75fb29..8167f87a2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; From b502b6682b9c73ae5760967eaf7161b3e8523e9e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch <maximilian@mbosch.me> Date: Mon, 20 May 2019 19:17:18 +0200 Subject: [PATCH 040/433] doc: clarify that optional attrs in a function argument will be ignored unless specified In `args@{ a ? 1 }: /* ... */` the value `a` won't be a part of `args` unless it's specified when calling the function, the default value will be ignored in this case. My personal point of view is that this behavior is a matter of taste, at least I was pretty sure that unmatched arguments will be a part of `args@` while debugging some Nix code last week. I decided to add a warning to the docs which hopefully reduces the confusion of further Nix developers who thought the same about `args@`. --- .../expressions/language-constructs.xml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index f961ed921..923b5d8c4 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -217,7 +217,25 @@ but can also be written as: ellipsis(<literal>...</literal>) as you can access attribute names as <literal>a</literal>, using <literal>args.a</literal>, which was given as an additional attribute to the function. - </para></listitem> + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> </itemizedlist> From 22f2744afdfa048de7bd078ba9d24dafbcc7d7c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 23:05:08 +0200 Subject: [PATCH 041/433] Iterate over references --- src/libexpr/parser.y | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 97e1a6584..78a503907 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -83,7 +83,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, assert(!attrPath.empty()); // Checking attrPath validity. // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { @@ -106,18 +106,18 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, } // Expr insertion. // ========================== - if (i->symbol.set()) { + if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { // This attr path is already defined. However, if both // e and the expr pointed by the attr path are two attribute sets, // we want to merge them. // Otherwise, throw an error. - ExprAttrs* ae = dynamic_cast<ExprAttrs *>(e); - ExprAttrs* jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); + auto ae = dynamic_cast<ExprAttrs *>(e); + auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); if (jAttrs && ae) { - for (auto ad: ae->attrs) { - ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first); + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs[ad.first] = ad.second; From abdedcdb387628aaa4d9dc362b7e588bbc62baae Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei <dingxiangfei2009@gmail.com> Date: Wed, 29 May 2019 17:01:39 +0800 Subject: [PATCH 042/433] bump cpptoml to v0.1.1 --- src/cpptoml/cpptoml.h | 573 +++++++++++++++++++++++++++++------------- 1 file changed, 392 insertions(+), 181 deletions(-) diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h index 07de010b1..5a00da3b4 100644 --- a/src/cpptoml/cpptoml.h +++ b/src/cpptoml/cpptoml.h @@ -4,8 +4,8 @@ * @date May 2013 */ -#ifndef _CPPTOML_H_ -#define _CPPTOML_H_ +#ifndef CPPTOML_H +#define CPPTOML_H #include <algorithm> #include <cassert> @@ -84,11 +84,12 @@ class option return &value_; } - const T& value_or(const T& alternative) const + template <class U> + T value_or(U&& alternative) const { if (!empty_) return value_; - return alternative; + return static_cast<T>(std::forward<U>(alternative)); } private: @@ -295,13 +296,12 @@ struct valid_value_or_string_convertible }; template <class T> -struct value_traits<T, typename std:: - enable_if<valid_value_or_string_convertible<T>:: - value>::type> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> { - using value_type = typename std:: - conditional<valid_value<typename std::decay<T>::type>::value, - typename std::decay<T>::type, std::string>::type; + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; using type = value<value_type>; @@ -312,12 +312,11 @@ struct value_traits<T, typename std:: }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_floating_point< - typename std::decay<T>::type>::value>:: - type> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> { using value_type = typename std::decay<T>::type; @@ -330,11 +329,11 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_signed<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -356,11 +355,10 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_unsigned<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -395,10 +393,15 @@ struct array_of_trait<array> template <class T> inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); inline std::shared_ptr<array> make_array(); + +namespace detail +{ template <class T> inline std::shared_ptr<T> make_element(); +} + inline std::shared_ptr<table> make_table(); -inline std::shared_ptr<table_array> make_table_array(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); #if defined(CPPTOML_NO_RTTI) /// Base type used to store underlying data type explicitly if RTTI is disabled @@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base> #if defined(CPPTOML_NO_RTTI) base_type type() const { - return type_; + return type_; } protected: @@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as() if (type() == base_type::INT) { auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); - return make_value<double>(static_cast<double>(v->get()));; + return make_value<double>(static_cast<double>(v->get())); } #else if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) @@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const { #if defined(CPPTOML_NO_RTTI) if (type() == base_type::FLOAT) - return std::static_pointer_cast<const value<double>>(shared_from_this()); + return std::static_pointer_cast<const value<double>>( + shared_from_this()); if (type() == base_type::INT) { @@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<array> make_element<array>() { return make_array(); } +} // namespace detail /** * Obtains a option<vector<T>>. The option will be empty if the array @@ -1060,7 +1067,7 @@ class table; class table_array : public base { friend class table; - friend std::shared_ptr<table_array> make_table_array(); + friend std::shared_ptr<table_array> make_table_array(bool); public: std::shared_ptr<base> clone() const override; @@ -1152,14 +1159,25 @@ class table_array : public base array_.reserve(n); } + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + private: #if defined(CPPTOML_NO_RTTI) - table_array() : base(base_type::TABLE_ARRAY) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) { // nothing } #else - table_array() + table_array(bool is_inline = false) : is_inline_(is_inline) { // nothing } @@ -1169,26 +1187,30 @@ class table_array : public base table_array& operator=(const table_array& rhs) = delete; std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; }; -inline std::shared_ptr<table_array> make_table_array() +inline std::shared_ptr<table_array> make_table_array(bool is_inline) { struct make_shared_enabler : public table_array { - make_shared_enabler() + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) { // nothing } }; - return std::make_shared<make_shared_enabler>(); + return std::make_shared<make_shared_enabler>(is_inline); } +namespace detail +{ template <> inline std::shared_ptr<table_array> make_element<table_array>() { - return make_table_array(); + return make_table_array(true); } +} // namespace detail // The below are overloads for fetching specific value types out of a value // where special casting behavior (like bounds checking) is desired @@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<table> make_element<table>() { return make_table(); } +} // namespace detail template <class T> std::shared_ptr<base> value<T>::clone() const @@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const inline std::shared_ptr<base> table_array::clone() const { - auto result = make_table_array(); + auto result = make_table_array(is_inline()); result->reserve(array_.size()); for (const auto& ptr : array_) result->array_.push_back(ptr->clone()->as_table()); @@ -1738,6 +1763,11 @@ inline bool is_number(char c) return c >= '0' && c <= '9'; } +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + /** * Helper object for consuming expected characters. */ @@ -1766,6 +1796,13 @@ class consumer [&](char c) { (*this)(c); }); } + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + int eat_digits(int len) { int val = 0; @@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line) line.push_back(static_cast<char>(c)); } } -} +} // namespace detail /** * The parser class. @@ -1914,21 +1951,25 @@ class parser std::string full_table_name; bool inserted = false; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table name"); if (!full_table_name.empty()) - full_table_name += "."; + full_table_name += '.'; full_table_name += part; if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif if (b->is_table()) curr_table = static_cast<table*>(b.get()); else if (b->is_table_array()) @@ -1946,16 +1987,23 @@ class parser curr_table->insert(part, make_table()); curr_table = static_cast<table*>(curr_table->get(part).get()); } - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); if (it == end) throw_parse_exception( "Unterminated table declaration; did you forget a ']'?"); + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + // table already existed if (!inserted) { @@ -1969,8 +2017,9 @@ class parser // since it has already been defined. If there aren't any // values, then it was implicitly created by something like // [a.b] - if (curr_table->empty() || std::any_of(curr_table->begin(), - curr_table->end(), is_value)) + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) { throw_parse_exception("Redefinition of table " + full_table_name); @@ -1989,36 +2038,45 @@ class parser if (it == end || *it == ']') throw_parse_exception("Table array name cannot be empty"); - std::string full_ta_name; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table array name"); if (!full_ta_name.empty()) - full_ta_name += "."; + full_ta_name += '.'; full_ta_name += part; - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif // if this is the end of the table array name, add an - // element to the table array that we just looked up + // element to the table array that we just looked up, + // provided it was not declared inline if (it != end && *it == ']') { if (!b->is_table_array()) + { throw_parse_exception("Key " + full_ta_name + " is not a table array"); + } + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + v->get().push_back(make_table()); curr_table = v->get().back().get(); } @@ -2059,15 +2117,16 @@ class parser = static_cast<table*>(curr_table->get(part).get()); } } - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); // consume the last "]]" - if (it == end) + auto eat = make_consumer(it, end, [this]() { throw_parse_exception("Unterminated table array name"); - ++it; - if (it == end) - throw_parse_exception("Unterminated table array name"); - ++it; + }); + eat(']'); + eat(']'); consume_whitespace(it, end); eol_or_comment(it, end); @@ -2076,7 +2135,35 @@ class parser void parse_key_value(std::string::iterator& it, std::string::iterator& end, table* curr_table) { - auto key = parse_key(it, end, [](char c) { return c == '='; }); + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + if (curr_table->contains(key)) throw_parse_exception("Key " + key + " already present"); if (it == end || *it != '=') @@ -2087,18 +2174,57 @@ class parser consume_whitespace(it, end); } - template <class Function> - std::string parse_key(std::string::iterator& it, - const std::string::iterator& end, Function&& fun) + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) { consume_whitespace(it, end); - if (*it == '"') + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') { - return parse_quoted_key(it, end); + return string_literal(it, end, *it); } else { - auto bke = std::find_if(it, end, std::forward<Function>(fun)); + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); return parse_bare_key(it, bke); } } @@ -2142,12 +2268,6 @@ class parser return key; } - std::string parse_quoted_key(std::string::iterator& it, - const std::string::iterator& end) - { - return string_literal(it, end, '"'); - } - enum class parse_type { STRING = 1, @@ -2193,7 +2313,7 @@ class parser parse_type determine_value_type(const std::string::iterator& it, const std::string::iterator& end) { - if(it == end) + if (it == end) { throw_parse_exception("Failed to parse value type"); } @@ -2209,7 +2329,11 @@ class parser { return *dtype; } - else if (is_number(*it) || *it == '-' || *it == '+') + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) { return determine_number_type(it, end); } @@ -2235,6 +2359,13 @@ class parser auto check_it = it; if (*check_it == '-' || *check_it == '+') ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + while (check_it != end && is_number(*check_it)) ++check_it; if (check_it != end && *check_it == '.') @@ -2283,57 +2414,56 @@ class parser bool consuming = false; std::shared_ptr<value<std::string>> ret; - auto handle_line - = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); - // whole line is whitespace - if (local_it == local_end) - return; - } + // whole line is whitespace + if (local_it == local_end) + return; + } - consuming = false; + consuming = false; - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } - ss << parse_escape_code(local_it, local_end); - continue; - } + ss << parse_escape_code(local_it, local_end); + continue; + } - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value<std::string>(ss.str()); - break; - } - } + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } - ss << *local_it++; - } - }; + ss << *local_it++; + } + }; // handle the remainder of the current line handle_line(it, end); @@ -2514,17 +2644,13 @@ class parser return value; } - bool is_hex(char c) - { - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - uint32_t hex_to_digit(char c) { if (is_number(c)) return static_cast<uint32_t>(c - '0'); - return 10 + static_cast<uint32_t>( - c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); } std::shared_ptr<base> parse_number(std::string::iterator& it, @@ -2538,25 +2664,6 @@ class parser ++check_it; }; - eat_sign(); - - auto eat_numbers = [&]() { - auto beg = check_it; - while (check_it != end && is_number(*check_it)) - { - ++check_it; - if (check_it != end && *check_it == '_') - { - ++check_it; - if (check_it == end || !is_number(*check_it)) - throw_parse_exception("Malformed number"); - } - } - - if (check_it == beg) - throw_parse_exception("Malformed number"); - }; - auto check_no_leading_zero = [&]() { if (check_it != end && *check_it == '0' && check_it + 1 != check_end && check_it[1] != '.') @@ -2565,7 +2672,80 @@ class parser } }; + auto eat_digits = [&](bool (*check_char)(char)) { + auto beg = check_it; + while (check_it != end && check_char(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !check_char(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') + { + eat_hex(); + return parse_int(it, check_it, 16); + } + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + + eat_sign(); check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + eat_numbers(); if (check_it != end @@ -2604,14 +2784,17 @@ class parser } std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, - const std::string::iterator& end) + const std::string::iterator& end, + int base = 10, + const char* prefix = "") { std::string v{it, end}; + v = prefix + v; v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); it = end; try { - return make_value<int64_t>(std::stoll(v)); + return make_value<int64_t>(std::stoll(v, nullptr, base)); } catch (const std::invalid_argument& ex) { @@ -2674,18 +2857,33 @@ class parser std::string::iterator find_end_of_number(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { + auto ret = std::find_if(it, end, [](char c) { return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+'; + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; } std::string::iterator find_end_of_date(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' - && c != '+' && c != '.'; + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; }); } @@ -2754,7 +2952,7 @@ class parser if (it == date_end) return make_value(ldate); - eat('T'); + eat.eat_or('T', ' '); local_datetime ldt; static_cast<local_date&>(ldt) = ldate; @@ -2850,9 +3048,9 @@ class parser auto arr = make_array(); while (it != end && *it != ']') { - auto value = parse_value(it, end); - if (auto v = value->as<Value>()) - arr->get().push_back(value); + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); else throw_parse_exception("Arrays must be homogeneous"); skip_whitespace_and_comments(it, end); @@ -2871,7 +3069,7 @@ class parser std::string::iterator& it, std::string::iterator& end) { - auto arr = make_element<Object>(); + auto arr = detail::make_element<Object>(); while (it != end && *it != ']') { @@ -2881,7 +3079,7 @@ class parser arr->get().push_back(((*this).*fun)(it, end)); skip_whitespace_and_comments(it, end); - if (*it != ',') + if (it == end || *it != ',') break; ++it; @@ -2906,8 +3104,11 @@ class parser throw_parse_exception("Unterminated inline table"); consume_whitespace(it, end); - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } } while (*it == ','); if (it == end || *it != '}') @@ -2987,7 +3188,8 @@ class parser if (it[4] != '-' || it[7] != '-') return {}; - if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) { // datetime type auto time_end = find_end_of_time(it + 11, date_end); @@ -3243,7 +3445,7 @@ class toml_writer { res += "\\\\"; } - else if ((const uint32_t)*it <= 0x001f) + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) { res += "\\u"; std::stringstream ss; @@ -3274,12 +3476,21 @@ class toml_writer */ void write(const value<double>& v) { - std::ios::fmtflags flags{stream_.flags()}; + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); - stream_ << std::showpoint; - write(v.get()); + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); - stream_.flags(flags); + stream_ << double_str; + has_naked_endline_ = false; } /** @@ -3287,9 +3498,9 @@ class toml_writer * offset_datetime. */ template <class T> - typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, - local_datetime, - offset_datetime>::value>::type + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type write(const value<T>& v) { write(v.get()); @@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a) a.accept(writer); return stream; } -} -#endif +} // namespace cpptoml +#endif // CPPTOML_H From cfd74aef1ef98e8830b18a300057dc8b80df2a9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 12:12:02 +0200 Subject: [PATCH 043/433] Fix eval-okay-fromTOML test Turns out we were mis-parsing single-quoted attributes, e.g. 'key2'. --- tests/lang/eval-okay-fromTOML.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 5b9d47122..10ad2fe99 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] From 17ef3e6f411b2cf1fa87593ad84bc2a94a1edd93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 12:22:52 +0200 Subject: [PATCH 044/433] Enable more fromTOML tests cpptoml now parses almost all examples from the spec. --- tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 10ad2fe99..392ff7a72 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 8e7cbd1c6..5626ef338 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -46,15 +46,15 @@ "character encoding" = "value" "ʎǝʞ" = "value" 'key2' = "value" - #'quoted "value"' = "value" + 'quoted "value"' = "value" name = "Orange" - # FIXME: cpptoml doesn't handle dotted keys properly yet. - #physical.color = "orange" - #physical.shape = "round" - #site."google.com" = true + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + # This is legal according to the spec, but cpptoml doesn't handle it. #a.b.c = 1 #a.d = 2 @@ -68,16 +68,14 @@ int6 = 5_349_221 int7 = 1_2_3_4_5 - # FIXME: cpptoml doesn't support these yet: + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef - #hex1 = 0xDEADBEEF - #hex2 = 0xdeadbeef - #hex3 = 0xdead_beef + oct1 = 0o01234567 + oct2 = 0o755 - #oct1 = 0o01234567 - #oct2 = 0o755 - - #bin1 = 0b11010110 + bin1 = 0b11010110 flt1 = +1.0 flt2 = 3.1415 @@ -126,8 +124,8 @@ key1 = "another string" key2 = 456 - #[dog."tater.man"] - #type.name = "pug" + [dog."tater.man"] + type.name = "pug" [a.b.c] [ d.e.f ] @@ -137,7 +135,7 @@ name = { first = "Tom", last = "Preston-Werner" } point = { x = 1, y = 2 } - #animal = { type.name = "pug" } + animal = { type.name = "pug" } [[products]] name = "Hammer" From a8251ba2ed9e833ef3e8b20d233064aba74446db Mon Sep 17 00:00:00 2001 From: Johannes Climacus <johannesclimacus@tuta.io> Date: Sat, 25 May 2019 05:06:43 -0400 Subject: [PATCH 045/433] Replace `type` with `command -v` in install script In POSIX sh, `type` is undefined. cf. https://pubs.opengroup.org/onlinepubs/9699919799/utilities/command.html#tag_20_22_04 --- scripts/install.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 7bff7b216..4857638c0 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -18,7 +18,7 @@ cleanup() { trap cleanup EXIT INT QUIT TERM require_util() { - type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + command -v "$1" > /dev/null 2>&1 || oops "you do not have '$1' installed, which I need to $2" } @@ -41,11 +41,11 @@ require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" -if type sha256sum > /dev/null 2>&1; then +if command -v sha256sum > /dev/null 2>&1; then hash2="$(sha256sum -b "$tarball" | cut -c1-64)" -elif type shasum > /dev/null 2>&1; then +elif command -v shasum > /dev/null 2>&1; then hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" -elif type openssl > /dev/null 2>&1; then +elif command -v openssl > /dev/null 2>&1; then hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" else oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" From 3b1cc8b0cbc0a2530e4671231257a497db8eede2 Mon Sep 17 00:00:00 2001 From: ln-nl <44435420+ln-nl@users.noreply.github.com> Date: Thu, 30 May 2019 18:08:33 +0300 Subject: [PATCH 046/433] release-common: fix build with `config.allowAliases = false` ```sh > nix build -f release.nix build.x86_64-linux error: undefined variable 'docbook5_xsl' at /vcs/nix/release-common.nix:45:7 ``` --- release-common.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index 4c5565985..2e8a951b9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -42,7 +42,7 @@ rec { libxml2 libxslt docbook5 - docbook5_xsl + docbook_xsl_ns autoconf-archive autoreconfHook ]; From 2d34028b1e4e46d600926d4ad0c687c4d1d58c75 Mon Sep 17 00:00:00 2001 From: worldofpeace <worldofpeace@protonmail.ch> Date: Fri, 31 May 2019 22:19:46 -0400 Subject: [PATCH 047/433] Add .github/FUNDING.yml --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..bbaabf93c --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: https://nixos.org/nixos/foundation.html From aec545c20b15eace71c6733107dd6fde7736afbe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Sat, 1 Jun 2019 15:27:43 +0200 Subject: [PATCH 048/433] Fix segfault in builtin fetchurl with hashed mirrors + SRI hashes --- src/libstore/builtins/fetchurl.cc | 3 ++- tests/fetchurl.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 92aec63a0..b1af3b4fc 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index ec3399b08..7319ced2b 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,6 +42,10 @@ ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +# Test hashed mirrors with an SRI hash. +nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + # Test unpacking a NAR. rm -rf $TEST_ROOT/archive mkdir -p $TEST_ROOT/archive From 4b0d6133836113572418f383ddd14c92bfebc768 Mon Sep 17 00:00:00 2001 From: JorisE <JorisE@users.noreply.github.com> Date: Tue, 4 Jun 2019 14:12:03 +0200 Subject: [PATCH 049/433] Minor typo --- doc/manual/expressions/language-constructs.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index 923b5d8c4..0d0cbbe15 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para> <simplesect xml:id="sect-let-expressions"><title>Let-expressions -A let-expression allows you define local variables for an +A let-expression allows you to define local variables for an expression. For instance, From 4a3e96281d6b9ae3dbc20e638f389677df9a649e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 5 Jun 2019 00:40:45 -0400 Subject: [PATCH 050/433] Handle SIGWINCH in main thread For the SIGWINCH signal to be caught, it needs to be set in sigaction on the main thread. Previously, this was broken, and updateWindowSize was never being called. Tested on macOS 10.14. --- src/libmain/shared.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..f2aeec9e1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,6 +125,9 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + /* Make sure SIGWINCH is handled as well. */ + if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); From 5011a52cf3f291c22d784aff6f5bbf8b99ae186a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 5 Jun 2019 20:18:47 -0400 Subject: [PATCH 051/433] Just enable hack on macOS This is not needed on linux at all! Tried to explain as much as I understand with the problem. --- src/libmain/shared.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f2aeec9e1..a6101342a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,8 +125,14 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); - /* Make sure SIGWINCH is handled as well. */ - if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); From 9e0f5f803f6cbfe9925cef69a0e58cbf5375bfaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Sat, 8 Jun 2019 00:41:19 +0200 Subject: [PATCH 052/433] Daemon: warn when an untrusted user cannot override a setting In a daemon-based Nix setup, some options cannot be overridden by a client unless the client's user is considered trusted. Currently, if an untrusted user tries to override one of those options, we are silently ignoring it. This can be pretty confusing in certain situations. e.g. a user thinks he disabled the sandbox when in reality he did not. We are now sending a warning message letting know the user some options have been ignored. Related to #1761. --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8d63b8f36..973f64a34 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref store, else if (setSubstituters(settings.extraSubstituters)) ; else - debug("ignoring untrusted setting '%s'", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name); } catch (UsageError & e) { warn(e.what()); } From 34fa8ce9179b17cc4cd13ae49b69fccf393271a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Jun 2019 16:34:06 +0200 Subject: [PATCH 053/433] nix: Support -j flag --- src/libmain/common-args.cc | 9 +++++++++ src/libmain/shared.cc | 4 ---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..cd752f467 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -175,10 +175,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description("build from source if substitution fails") .set(&(bool&) settings.tryFallback, true); - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); - auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { mkFlag(shortName, longName, description, [=](unsigned int n) { From 5064971ded3e2cc7c6b80d3166270a87a9650c09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 21:15:45 +0200 Subject: [PATCH 054/433] Fix test failures when $TMPDIR changes (cherry picked from commit c38c726eb5d447c7e9d894d57cd05ac46c173ddd) --- tests/build-dry.sh | 6 +++--- tests/nix-copy-ssh.sh | 2 +- tests/nix-shell.sh | 8 ++++---- tests/placeholders.sh | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c..e72533e70 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a4..eb801548d 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399..ee502dddb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc..cd1bb7bc2 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY From 26bc876ae6e7ceff8b37ba5c2c1043ce216cf384 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Jun 2019 16:45:00 +0200 Subject: [PATCH 055/433] nix: Add -L alias for --print-build-logs --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 4f87ad72b..25e321b86 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -45,6 +45,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("print-build-logs") + .shortName('L') .description("print full build logs on stderr") .set(&printBuildLogs, true); From b693029ca076abf8a4084a053a122ffcdf4d61b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 16 Jun 2019 09:43:20 +0200 Subject: [PATCH 056/433] Style fix --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 973f64a34..e88aaf636 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref store, else if (setSubstituters(settings.extraSubstituters)) ; else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } From e84c26564507388f2e8b7c7f2b00eb2b57856da1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 May 2019 22:29:15 +0200 Subject: [PATCH 057/433] Run builds in a pseudo-terminal This allows many programs (e.g. gcc, clang, cmake) to print colorized log output (assuming $TERM is set to a value like "xterm"). There are other ways to get colors, in particular setting CLICOLOR_FORCE, but they're less widely supported and can break programs that parse tool output. --- src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0bd738809..9de7613f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -1558,8 +1559,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -2181,7 +2182,43 @@ void DerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -4361,14 +4398,15 @@ void Worker::waitForInput() for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { ssize_t rd = read(k, buffer.data(), buffer.size()); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); From 82ca6ef390b752faf1bf27b22daa34b29a4b00d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 19 May 2019 16:56:08 +0200 Subject: [PATCH 058/433] Set $TERM --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9de7613f8..5a75403cf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2443,6 +2443,9 @@ void DerivationGoal::initEnv() may change that in the future. So tell the builder which file descriptor to use for that. */ env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } From 2743bf0bb1727056ce6d6d6be4e38c3251aac1f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 16 Jun 2019 20:02:40 +0200 Subject: [PATCH 059/433] Hopefully fix macOS tests --- src/libstore/build.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5a75403cf..cccf0c7c6 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2190,11 +2190,13 @@ void DerivationGoal::startBuilder() std::string slaveName(ptsname(builderOut.readSide.get())); - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); - if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } #if 0 // Mount the pt in the sandbox so that the "tty" command works. From 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:08:04 +0200 Subject: [PATCH 060/433] Another attempt at getting pseudoterminals to work on macOS --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cccf0c7c6..5b38bcf3c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2196,6 +2196,9 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); } #if 0 From 74a65d313f5c9f4d504ea9e2caadfe0ff83776d5 Mon Sep 17 00:00:00 2001 From: Bruno Bieth Date: Thu, 20 Jun 2019 14:47:45 +0200 Subject: [PATCH 061/433] Nix uses the CPP SDK, not Java --- doc/manual/packages/s3-substituter.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index e7589ffdb..1722090ef 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -113,7 +113,7 @@ the S3 URL: exactly s3://example-nix-cache. Nix will use the default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain for authenticating requests to Amazon S3. @@ -138,7 +138,7 @@ the S3 URL: be s3://example-nix-cache. Nix will use the default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain for authenticating requests to Amazon S3. From 99cec651c9c1d95656e182871f0bb186f798077c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 16:52:11 +0200 Subject: [PATCH 062/433] Add more fromTOML tests --- tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 392ff7a72..3c16e1fa1 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 5626ef338..dd9ad9ea1 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -147,6 +147,25 @@ name = "Nail" sku = 284758393 color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" '') (builtins.fromTOML '' From 94f11d0a61bba15ff8a93f2de8d1883783c8c508 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 17:09:09 +0200 Subject: [PATCH 063/433] Fix abort in fromTOML Fixes #2969. --- src/libexpr/primops/fromTOML.cc | 13 +++++++++++++ tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4128de05d..a84e569e9 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); } + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + else if (t->is_value()) { if (auto val = t->as()) mkInt(v, val->get()); diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 3c16e1fa1..d0dd3af2c 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index dd9ad9ea1..963932689 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -198,4 +198,11 @@ "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + ] From dc29e9fb47f9f98a851dc88b2bd3cae4b5c1fe6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:20:50 +0800 Subject: [PATCH 064/433] downloadCached: Return ETag (cherry picked from commit 529add316c5356a8060c35f987643b7bf5c796dc) --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/download.cc | 17 +++++++++++++---- src/libstore/download.hh | 12 ++++++++++-- src/nix-channel/nix-channel.cc | 6 +++--- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..37c74a94b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); + return getDownloader()->downloadCached(state.store, s, true).path; else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78a503907..29860fb05 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -677,7 +677,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 06f577f36..55a1bde11 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2083,7 +2083,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 22382ab1d..cb77cdc77 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -790,7 +790,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -802,8 +802,11 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Path expectedStorePath; if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -822,6 +825,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -833,6 +838,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa skip = true; if (effectiveUrl) *effectiveUrl = url_; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -850,6 +856,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa auto res = download(request); if (effectiveUrl) *effectiveUrl = res.effectiveUrl; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; @@ -871,6 +878,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } @@ -904,7 +912,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa url, expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..8acfe4e1a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -41,6 +41,12 @@ struct DownloadResult uint64_t bodySize = 0; }; +struct CachedDownloadResult +{ + Path path; + std::optional etag; +}; + class Store; struct Downloader @@ -64,8 +70,10 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached( + ref store, const string & uri, bool unpack, string name = "", + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, + int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..7b23088a2 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -88,7 +88,7 @@ static void update(const StringSet & channelNames) // definition from a consistent location if the redirect changes mid-download. std::string effectiveUrl; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); + auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; url = chomp(std::move(effectiveUrl)); // If the URL contains a version number, append it to the name @@ -123,10 +123,10 @@ static void update(const StringSet & channelNames) // Download the channel tarball. auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } catch (DownloadError & e) { fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } chomp(filename); } From b43e1e186e50bedc30d05d063dffbd751c1ff161 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 May 2019 15:38:24 +0200 Subject: [PATCH 065/433] CachedDownloadResult: Include store path Also, make fetchGit and fetchMercurial update allowedPaths properly. (Maybe the evaluator, rather than the caller of the evaluator, should apply toRealPath(), but that's a bigger change.) (cherry picked from commit 5c34d665386f4053d666b0899ecca0639e500fbd) --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libstore/download.cc | 2 ++ src/libstore/download.hh | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index aaf02c856..cf7ccca41 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -235,7 +235,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374..d576138f5 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -214,7 +214,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cb77cdc77..975cfd97d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -804,6 +804,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; + result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); return result; } @@ -912,6 +913,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & url, expectedHash.to_string(), gotHash.to_string()); } + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 8acfe4e1a..aa8c34be2 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -43,6 +43,9 @@ struct DownloadResult struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; Path path; std::optional etag; }; From 2fef4dd29673ef383b695480c9b8212f2e8d4711 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:06:37 +0200 Subject: [PATCH 066/433] Downloader: Propagate exceptions from decompressionSink->finish() (cherry picked from commit 15fa70cd1b853f5e62662b99ccb9ef3da6cfadff) --- src/libstore/download.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 975cfd97d..8e94646d4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -327,8 +327,13 @@ struct CurlDownloader : public Downloader debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; From 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:48:52 +0200 Subject: [PATCH 067/433] Fix 'error 9 while decompressing xz file' Once we've started writing data to a Sink, we can't restart a download request, because then we end up writing duplicate data to the Sink. Therefore we shouldn't handle retries in Downloader but at a higher level (in particular, in copyStorePath()). Fixes #2952. (cherry picked from commit a67cf5a3585c41dd9f219a2c7aa9cf67fa69520b) --- src/libstore/binary-cache-store.cc | 18 +++--- src/libstore/download.cc | 67 ++++---------------- src/libstore/download.hh | 11 +++- src/libstore/http-binary-cache-store.cc | 55 +++++++++++----- src/libstore/store-api.cc | 84 +++++++++++++------------ src/libutil/retry.hh | 38 +++++++++++ src/libutil/types.hh | 2 + 7 files changed, 156 insertions(+), 119 deletions(-) create mode 100644 src/libutil/retry.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..8b736056e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,8 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "retry.hh" +#include "download.hh" #include @@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; + return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + }); } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8e94646d4..43d231e30 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "retry.hh" #ifdef ENABLE_S3 #include @@ -19,11 +20,9 @@ #include #include -#include #include #include #include -#include #include using namespace std::string_literals; @@ -62,9 +61,6 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; - std::random_device rd; - std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -77,12 +73,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; std::string encoding; @@ -401,9 +391,7 @@ struct CurlDownloader : public Downloader } } - attempt++; - - auto exc = + fail( code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -414,31 +402,15 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); + request.verb(), request.uri, curl_easy_strerror(code), code))); } } }; struct State { - struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { - return i1->embargo > i2->embargo; - } - }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::vector> incoming; }; Sync state_; @@ -451,7 +423,6 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() - : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -545,9 +516,7 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ + /* Add new curl requests from the incoming requests queue. */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -556,22 +525,9 @@ struct CurlDownloader : public Downloader } std::vector> incoming; - auto now = std::chrono::steady_clock::now(); - { auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } + std::swap(state->incoming, incoming); quit = state->quit; } @@ -598,7 +554,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); + state->incoming.clear(); state->quit = true; } } @@ -614,7 +570,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); + state->incoming.push_back(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -697,7 +653,9 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return enqueueDownload(request).get(); + return retry(request.tries, [&]() { + return enqueueDownload(request).get(); + }); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -883,7 +841,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("%s; using cached result", e.msg()); result.etag = expectedETag; } } @@ -933,5 +891,4 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } - } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index aa8c34be2..752f91857 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -62,11 +62,13 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. */ + /* Synchronously download a file. The request will be retried in + case of transient failures. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + invoked on the thread of the caller. The request will not be + retried in case of transient failures. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -95,6 +97,11 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } + + bool isTransient() override + { + return error == Downloader::Error::Transient; + } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..5133dba57 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "retry.hh" namespace nix { @@ -114,7 +115,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -137,21 +137,46 @@ protected: { checkEnabled(); - auto request(makeRequest(path)); + struct State + { + DownloadRequest request; + std::function tryDownload; + unsigned int attempt = 0; + State(DownloadRequest && request) : request(request) {} + }; - getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - maybeDisable(); - callback.rethrow(); - } catch (...) { - callback.rethrow(); - } - }}); + auto state = std::make_shared(makeRequest(path)); + + state->tryDownload = [callback, state, this]() { + getDownloader()->enqueueDownload(state->request, + {[callback, state, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + ++state->attempt; + if (state->attempt < state->request.tries && e.isTransient()) { + auto ms = retrySleepTime(state->attempt); + warn("%s; retrying in %d ms", e.what(), ms); + /* We can't sleep here because that would + block the download thread. So use a + separate thread for sleeping. */ + std::thread([state, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + state->tryDownload(); + }).detach(); + } else { + maybeDisable(); + callback.rethrow(); + } + } catch (...) { + callback.rethrow(); + } + }}); + }; + + state->tryDownload(); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..28ad7c019 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,10 +6,11 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "retry.hh" +#include "download.hh" #include - namespace nix { @@ -572,54 +573,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + retry(downloadSettings.tries, [&]() { - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); - auto info = srcStore->queryPathInfo(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); - uint64_t total = 0; + auto info = srcStore->queryPathInfo(storePath); - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; + uint64_t total = 0; - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); + }); } diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh new file mode 100644 index 000000000..b45cb37f7 --- /dev/null +++ b/src/libutil/retry.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "logging.hh" + +#include +#include +#include +#include + +namespace nix { + +inline unsigned int retrySleepTime(unsigned int attempt) +{ + std::random_device rd; + std::mt19937 mt19937; + return 250.0 * std::pow(2.0f, + attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); +} + +template +C retry(unsigned int attempts, std::function && f) +{ + unsigned int attempt = 0; + while (true) { + try { + return f(); + } catch (BaseError & e) { + ++attempt; + if (attempt >= attempts || !e.isTransient()) + throw; + auto ms = retrySleepTime(attempt); + warn("%s; retrying in %d ms", e.what(), ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 92bf469b5..88e3243f4 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,6 +109,8 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); + + virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ From 7b9c68766d513260d5262d5782b46384834cdb33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:43:45 +0200 Subject: [PATCH 068/433] Add '--no-net' convenience flag This flag * Disables substituters. * Sets the tarball-ttl to infinity (ensuring e.g. that the flake registry and any downloaded flakes are considered current). * Disables retrying downloads and sets the connection timeout to the minimum. (So it doesn't completely disable downloads at the moment.) (cherry picked from commit 8ea842260b4fd93315d35c5ba94b1ff99ab391d8) --- src/libmain/common-args.cc | 11 +++++++++++ src/libstore/download.cc | 18 +----------------- src/libstore/download.hh | 23 ++++++++++++++++++++++- src/libstore/globals.hh | 2 +- src/libstore/http-binary-cache-store.cc | 1 - 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9e1d7cee6..0486932b2 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "globals.hh" +#include "download.hh" namespace nix { @@ -44,6 +45,16 @@ MixCommonArgs::MixCommonArgs(const string & programName) settings.set("max-jobs", s); }); + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([]() { + settings.useSubstitutes = false; + settings.tarballTtl = std::numeric_limits::max(); + downloadSettings.tries = 0; + downloadSettings.connectTimeout = 1; + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 43d231e30..342a8aa21 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -29,23 +29,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 752f91857..dae082ab9 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,34 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..0af8215d1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,7 +255,7 @@ public: "Secret keys with which to sign local builds."}; Setting tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5133dba57..5633b4355 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,7 +85,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { From f8b30338ac231262bdf19844f044f0572c460048 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 23:36:29 +0200 Subject: [PATCH 069/433] Refactor downloadCached() interface (cherry picked from commit df3f5a78d5ab0a1f2dc9d288b271b38a9b8b33b5) --- src/libexpr/common-eval-args.cc | 8 +++--- src/libexpr/parser.y | 4 ++- src/libexpr/primops.cc | 22 ++++++++--------- src/libstore/download.cc | 44 ++++++++++++++++----------------- src/libstore/download.hh | 20 +++++++++++---- src/nix-channel/nix-channel.cc | 14 +++++------ 6 files changed, 63 insertions(+), 49 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 37c74a94b..13950ab8d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true).path; - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 29860fb05..967c88d9b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -677,7 +677,9 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 55a1bde11..070e72f3a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 342a8aa21..892c1b21d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -293,10 +293,10 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); @@ -737,18 +737,20 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; result.storePath = expectedStorePath; @@ -782,10 +784,9 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); @@ -799,18 +800,17 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -830,7 +830,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -853,11 +853,11 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + url, request.expectedHash.to_string(), gotHash.to_string()); } result.storePath = storePath; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index dae082ab9..9e965b506 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -57,11 +57,23 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + struct CachedDownloadResult { // Note: 'storePath' may be different from 'path' when using a @@ -69,6 +81,7 @@ struct CachedDownloadResult Path storePath; Path path; std::optional etag; + std::string effectiveUri; }; class Store; @@ -96,10 +109,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached( - ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, - int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 7b23088a2..bd1371dba 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -121,12 +123,10 @@ static void update(const StringSet & channelNames) } // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } From 64ec087f582cec33733f4102ab42a3e8f291758a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 12:49:18 +0200 Subject: [PATCH 070/433] Fix 32-bit overflow with --no-net --no-net causes tarballTtl to be set to the largest 32-bit integer, which causes comparison like 'time + tarballTtl < other_time' to fail on 32-bit systems. So cast them to 64-bit first. https://hydra.nixos.org/build/95076624 (cherry picked from commit 29ccb2e9697ee2184012dd13854e487928ae4441) --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 13 +++++-------- src/libstore/download.cc | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index cf7ccca41..3dcf3e9ff 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -116,7 +116,7 @@ GitInfo exportGit(ref store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d576138f5..a907d0e1c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -96,17 +96,14 @@ HgInfo exportMercurial(ref store, const std::string & uri, try { runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); } - catch (ExecError & e){ + catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) - { + if (pathExists(transJournal)) { runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - else - { - throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 892c1b21d..6ce9525c3 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -784,7 +784,7 @@ CachedDownloadResult Downloader::downloadCached( auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; result.effectiveUri = request.uri; result.etag = ss[1]; From 5600b070a766b83200a68d3632793917cf19a550 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:12:03 +0200 Subject: [PATCH 071/433] Add "warning" verbosity level This ensures that "nix" shows warnings. Previously these were hidden because they were at "info" level. (cherry picked from commit 615a9d031d22a6aee64f8511e15685e47b6f8796) --- src/libutil/logging.cc | 3 ++- src/libutil/logging.hh | 1 + src/nix/main.cc | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae..b379306f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102..5f2219445 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/nix/main.cc b/src/nix/main.cc index 25e321b86..19bb7a543 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -93,7 +93,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlError; + verbosity = lvlWarn; settings.verboseBuild = false; NixArgs args; From 09dde33c19129b949a064d35e248a3e318178b08 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:57:22 +0200 Subject: [PATCH 072/433] Automatically use --no-net if there are no network interfaces (cherry picked from commit 04a59769963fe2a28d10ba15de743fe499333c80) --- src/libmain/common-args.cc | 11 -------- src/nix/main.cc | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 0486932b2..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,5 @@ #include "common-args.hh" #include "globals.hh" -#include "download.hh" namespace nix { @@ -45,16 +44,6 @@ MixCommonArgs::MixCommonArgs(const string & programName) settings.set("max-jobs", s); }); - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([]() { - settings.useSubstitutes = false; - settings.tarballTtl = std::numeric_limits::max(); - downloadSettings.tries = 0; - downloadSettings.connectTimeout = 1; - }); - std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/nix/main.cc b/src/nix/main.cc index 19bb7a543..73c4b8db1 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,19 +8,52 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include +#include +#include +#include + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) && + !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; + bool useNet = true; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -53,6 +86,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -108,6 +146,23 @@ void mainWrapped(int argc, char * * argv) startProgressBar(args.printBuildLogs); + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + args.command->prepare(); args.command->run(); } From 88571219d97f2bdfdbafcff25ef6ee424b0b008f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jun 2019 13:27:16 +0200 Subject: [PATCH 073/433] nix-channel: Don't fetch binary-cache-url This has been ignored since the Perl->C++ rewrite. --- corepkgs/unpack-channel.nix | 8 ++------ doc/manual/command-ref/nix-channel.xml | 23 +++++------------------ src/libstore/store-api.hh | 3 +-- src/nix-channel/nix-channel.cc | 9 --------- 4 files changed, 8 insertions(+), 35 deletions(-) diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index a654db40e..d39a20637 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -18,21 +18,17 @@ let if [ * != $channelName ]; then mv * $out/$channelName fi - if [ -n "$binaryCacheURL" ]; then - mkdir $out/binary-caches - echo -n "$binaryCacheURL" > $out/binary-caches/$channelName - fi ''; in -{ name, channelName, src, binaryCacheURL ? "" }: +{ name, channelName, src }: derivation { system = builtins.currentSystem; builder = shell; args = [ "-e" builder ]; - inherit name channelName src binaryCacheURL; + inherit name channelName src; PATH = "${nixBinDir}:${coreutils}"; diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a76..5a2866e6b 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,12 +31,11 @@ Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing both a set of Nix -expressions and a pointer to a binary cache. See also . +A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. See also . This command has the following operations: @@ -172,18 +171,6 @@ following files: - binary-cache-url - - A file containing the URL to a binary cache (such - as https://cache.nixos.org). Nix will automatically - check this cache for pre-built binaries, if the user has - sufficient rights to add binary caches. For instance, in a - multi-user Nix setup, the binary caches provided by the channels - of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored. - - - diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0f..599677376 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -766,8 +766,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list> getDefaultSubstituters(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index bd1371dba..06eb3d23b 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,15 +113,6 @@ static void update(const StringSet & channelNames) } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. try { filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; From 324a5dc92f8e50e6b637c5e67dea48c80be10837 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jun 2019 21:57:18 +0200 Subject: [PATCH 074/433] ProgressBar: Fix updating 'updateCV.notify_one()' does nothing if the update thread is not waiting for updateCV (in particular this happens when it is sleeping on quitCV). So also set a variable to ensure that the update isn't lost. --- src/nix/progress-bar.cc | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index e71045408..b1c1d87de 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -62,6 +62,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool haveUpdate = true; }; Sync state_; @@ -83,7 +84,8 @@ public: updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { - state.wait(updateCV); + if (!state->haveUpdate) + state.wait(updateCV); draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } @@ -178,7 +180,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(); + update(*state); } /* Check whether an activity has an ancestore with the specified @@ -213,7 +215,7 @@ public: state->its.erase(i); } - update(); + update(*state); } void result(ActivityId act, ResultType type, const std::vector & fields) override @@ -223,7 +225,7 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(); + update(*state); } else if (type == resBuildLogLine) { @@ -239,26 +241,26 @@ public: info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(); + update(*state); } } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(); + update(*state); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(); + update(*state); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(); + update(*state); } else if (type == resProgress) { @@ -269,7 +271,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(); + update(*state); } else if (type == resSetExpected) { @@ -281,17 +283,19 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(); + update(*state); } } - void update() + void update(State & state) { + state.haveUpdate = true; updateCV.notify_one(); } void draw(State & state) { + state.haveUpdate = false; if (!state.active) return; std::string line; From ec0087df0a2da5c68363b3f4509d4545deed97a0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 27 Jun 2019 14:22:53 -0400 Subject: [PATCH 075/433] =?UTF-8?q?Don=E2=80=99t=20use=20entire=20/etc/nss?= =?UTF-8?q?witch.conf=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default nsswitch.conf(5) file in most distros can handle many different things including host name, user names, groups, etc. In Nix, we want to limit the amount of impurities that come from these things. As a result, we should only allow nss to be used for gethostbyname(3) and getservent(3). /cc @Ericson2314 --- src/libstore/build.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5b38bcf3c..813d7e2c2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2728,7 +2728,13 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); if (pathExists("/var/run/nscd/socket")) From 6847c9278840edc97f1ef85b5fafac4338fb1b37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Jun 2019 15:38:23 +0200 Subject: [PATCH 076/433] Fix macOS build failure Issue #2976. --- src/nix/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 73c4b8db1..a80fd0ea6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -39,8 +39,8 @@ static bool haveInternet() return true; } } else if (i->ifa_addr->sa_family == AF_INET6) { - if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) && - !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr)) + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) return true; } } From ec58ba38c55a3ab61fdb1da7d746251e4887ea2c Mon Sep 17 00:00:00 2001 From: Aniket Deshpande Date: Fri, 28 Jun 2019 19:43:22 +0530 Subject: [PATCH 077/433] Fix `http2 = false` having no effect. Fixes #2971. Setting `http2 = false` in nix config (e.g. /etc/nix/nix.conf) had no effect, and `nix-env -vvvvv -i hello` still downloaded .nar packages using HTTP/2. In `src/libstore/download.cc`, the `CURL_HTTP_VERSION_2TLS` option was being explicitly set when `downloadSettings.enableHttp2` was `true`, but, `CURL_HTTP_VERSION_1_1` option was not being explicitly set when `downloadSettings.enableHttp2` was `false`. This may be because `https://curl.haxx.se/libcurl/c/libcurl-env.html` states: "You have to set this option if you want to use libcurl's HTTP/2 support." but, also, in the changelog, states: "DEFAULT Since curl 7.62.0: CURL_HTTP_VERSION_2TLS Before that: CURL_HTTP_VERSION_1_1" So, the default setting for `libcurl` is HTTP/2 for version >= 7.62.0. In this commit, option `CURLOPT_HTTP_VERSION` is explicitly set to `CURL_HTTP_VERSION_1_1` when `downloadSettings.enableHttp2` nix config setting is `false`. This can be tested by running `nix-env -vvvvv -i hello | grep HTTP` --- src/libstore/download.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 6ce9525c3..0c5a73ea3 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -244,6 +244,8 @@ struct CurlDownloader : public Downloader #if LIBCURL_VERSION_NUM >= 0x072f00 if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); From 97baf32fbca13101f58d30bb3a601302c3d560f3 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Tue, 2 Jul 2019 00:12:12 +0200 Subject: [PATCH 078/433] build: add exit code for hash and check mismatches Makes it easier to identify the failure reason in other tooling, eg. differentiate between a non-deterministic --check vs a failed build. $ nix-build '' --argstr url http://example.org --argstr sha256 0000000000000000000000000000000000000000000000000000 hash mismatch in fixed-output derivation '/nix/store/nzi9ck45rwlxzcwr25is7qlf3hs5xl83-example.org': wanted: sha256:0000000000000000000000000000000000000000000000000000 got: sha256:08y4734bm2zahw75b16bcmcg587vvyvh0n11gwiyir70divwp1rm $ echo $? 102 $ nix-build -E 'with import {}; runCommand "foo" {} "date +%s > $out"' --check warning: rewriting hashes in '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo'; cross fingers error: derivation '/nix/store/mggc8dz13ackb49qca6m23zq4fpq132q-foo.drv' may not be deterministic: output '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo' differs $ echo $? 104 --- src/libstore/build.cc | 16 +++++++++++++--- src/libstore/download.cc | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5b38bcf3c..c076ea8b0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -266,6 +266,12 @@ public: /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr hook; @@ -3213,6 +3219,7 @@ void DerivationGoal::registerOutputs() /* Throw an error after registering the path as valid. */ + worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", dest, h.to_string(), h2.to_string())); @@ -3255,6 +3262,7 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); @@ -3266,10 +3274,10 @@ void DerivationGoal::registerOutputs() buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") % drvPath % path); } @@ -4101,6 +4109,8 @@ Worker::Worker(LocalStore & store) lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } @@ -4461,7 +4471,7 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1))); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0c5a73ea3..7a2af237e 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -855,10 +855,11 @@ CachedDownloadResult Downloader::downloadCached( } if (expectedStorePath != "" && storePath != expectedStorePath) { + unsigned int statusCode = 102; Hash gotHash = request.unpack ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first : hashFile(request.expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, request.expectedHash.to_string(), gotHash.to_string()); } From cbf84bcce7d3fddb353d3aa0f012f5cbdbb77629 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 11 May 2019 23:14:19 +0200 Subject: [PATCH 079/433] build: use binary mask for build status flags If multiple builds with fail with different errors it will be reflected in the status code. eg. 103 => timeout + hash mismatch 105 => timeout + check mismatch 106 => hash mismatch + check mismatch 107 => timeout + hash mismatch + check mismatch --- src/libstore/build.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c076ea8b0..abfae3c7c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4471,7 +4471,15 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1))); + unsigned int mask = 0; + if (timedOut) + mask |= 1; + if (hashMismatch) + mask |= 2; + if (checkMismatch) + mask |= 4; + + return mask ? 100 + mask : 1; } From 99ee3755dd12e18f7085b8319a27798b0e5b7de9 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 11 May 2019 23:32:53 +0200 Subject: [PATCH 080/433] build: add tests for --check status codes --- tests/check.nix | 5 +++++ tests/check.sh | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/check.nix b/tests/check.nix index 08aac2fb0..56c82e565 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -10,6 +10,11 @@ with import ./config.nix; ''; }; + hashmismatch = import { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + fetchurl = import { url = "file://" + toString ./lang/eval-okay-xml.exp.xml; sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; diff --git a/tests/check.sh b/tests/check.sh index b05e40ffb..aa1e5cd26 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -6,14 +6,16 @@ nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link -(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +[ "$status" = "104" ] grep 'may not be deterministic' $TEST_ROOT/log clearStore nix-build dependencies.nix --no-out-link --repeat 3 -(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') @@ -23,10 +25,23 @@ echo foo > $path chmod -w $path nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' - # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' - [[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "106" ] From 1ac399dd115d5c86629389e0cdfefa0d654fc90a Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Mon, 13 May 2019 21:03:36 +0200 Subject: [PATCH 081/433] nix-store: document exit codes --- doc/manual/command-ref/nix-store.xml | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee..bf12d06f1 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -215,6 +215,41 @@ printed.) +Special exit codes: + + + + 100 + Generic build failure, the builder process + returned with a non-zero exit code. + + + 101 + Build timeout, the build was aborted because it + did not complete within the specified timeout. + + + + 102 + Hash mismatch, the build output was rejected + because it does not match the specified outputHash. + + + + 104 + Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable. + + + + + +With the flag it's possible for +multiple build failures to occur, in this case the 1xx status codes +are or combined. + From a52c331edba6e8c67469f53dda0be2903d18fa8c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 15 Jun 2019 15:28:32 +0200 Subject: [PATCH 082/433] build: replace 100 offset for build exit codes --- doc/manual/command-ref/nix-store.xml | 11 +++++++++-- src/libstore/build.cc | 24 +++++++++++++++++++----- tests/check.sh | 4 ++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index bf12d06f1..2ecc63db7 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -247,8 +247,15 @@ printed.) With the flag it's possible for -multiple build failures to occur, in this case the 1xx status codes -are or combined. +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index abfae3c7c..350ac4092 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4471,15 +4471,29 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 if (timedOut) - mask |= 1; + mask |= 0x01; // 101 if (hashMismatch) - mask |= 2; - if (checkMismatch) - mask |= 4; + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } - return mask ? 100 + mask : 1; + if (mask) + mask |= 0x60; + return mask ? mask : 1; } diff --git a/tests/check.sh b/tests/check.sh index aa1e5cd26..bc23a6634 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -7,8 +7,8 @@ nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? -[ "$status" = "104" ] grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] clearStore @@ -44,4 +44,4 @@ nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || # Multiple failures with --keep-going nix-build check.nix -A nondeterministic --no-out-link nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? -[ "$status" = "106" ] +[ "$status" = "110" ] From 17d3ec3405eb114f89dc42d065afa887161f6149 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 27 Jun 2019 14:21:22 -0400 Subject: [PATCH 083/433] checkStoreName: give more precise/verbose error information $ sudo ./inst/bin/nix-instantiate -E '"${./.git}"' error: The path name '.git' is invalid: it is illegal to start the name with a period. Path names are alphanumeric and can include the symbols +-._?= and must not begin with a period. Note: If '.git' is a source file and you cannot rename it on disk, builtins.path { name = ... } can be used to give it an alternative name. --- src/libstore/store-api.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28ad7c019..92f01fd2e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -86,18 +86,25 @@ string storePathToHash(const Path & path) void checkStoreName(const string & name) { string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") - throw Error(format("illegal name: '%1%'") % name); + throw Error(baseError % "it is illegal to start the name with a period"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { - throw Error(format("invalid character '%1%' in name '%2%'") - % i % name); + throw Error(baseError % (format("the '%1%' character is invalid") % i)); } } From c8205a3413217ccf8a6a1f7e064b06a5b86c3253 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 2 Jul 2019 09:04:04 -0400 Subject: [PATCH 084/433] builtins.fetchGit: document absolute ref support --- doc/manual/expressions/builtins.xml | 15 +++++++++++++++ doc/manual/release-notes/release-notes.xml | 1 + doc/manual/release-notes/rl-2.3.xml | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 doc/manual/release-notes/rl-2.3.xml diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 412622714..dcd7aaf24 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -419,6 +419,13 @@ stdenv.mkDerivation { … } This is often a branch or tag name. Defaults to HEAD. + + + By default, the ref value is prefixed + with refs/heads/. As of Nix 2.3.0 + Nix will not prefix refs/heads/ if + ref starts with refs/. + @@ -432,6 +439,14 @@ stdenv.mkDerivation { … } } + + Fetching an arbitrary ref + builtins.fetchGit { + url = "https://gitub.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +} + + Fetching a repository's specific commit on an arbitrary branch diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index e8ff586fa..2655d68e3 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,7 @@ --> + diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000..428213b36 --- /dev/null +++ b/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,22 @@ +
+ +Release 2.3 (????-??-??) + +This release contains the following changes: + + + + + builtins.fetchGit's ref + argument now allows specifying an absolute remote ref. + Nix will automatically prefix ref with + refs/heads only if ref doesn't + already begin with refs/. + + + +
From a3c77c15369703975f7c3e65ebb829580754bc1e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 14 Mar 2019 22:39:49 +0100 Subject: [PATCH 085/433] nix-store: document --add-fixed --- doc/manual/command-ref/nix-store.xml | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee..625598310 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -883,6 +883,60 @@ $ nix-store --add ./foo.c + + +Operation <option>--add-fixed</option> + +Synopsis + + + nix-store + + + algorithm + paths + + + + +Description + +The operation adds the specified paths to +the Nix store. Unlike paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. + + +This operation has the following options: + + + + + + + Use recursive instead of flat hashing mode, used when adding directories + to the store. + + + + + + + + + + +Example + + +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + + + + + From 68bdd83dc88ec55c6c51fa92e84e7d7d408c554a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 2 Jul 2019 11:18:36 -0400 Subject: [PATCH 086/433] timeout: test for error code --- tests/timeout.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/timeout.sh b/tests/timeout.sh index 39ecf0a1a..eea9b5731 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -2,10 +2,14 @@ source common.sh -failed=0 -messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" -if [ $failed -ne 0 ]; then - echo "error: 'nix-store' succeeded; should have timed out" + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi From 96cd3d607374cb4bab27095a62a6f3a832518c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 03:11:52 +0200 Subject: [PATCH 087/433] autoconf: Change quotes in description. The unbalanced single-quotes cause many editor syntax highlighters to interpret the rest of the file as a string literal, making it easier to make syntax mistakes in absence of proper highlighting. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f5b1614f1..398bac047 100644 --- a/configure.ac +++ b/configure.ac @@ -42,7 +42,7 @@ esac AC_MSG_RESULT($system) AC_SUBST(system) -AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. From 00a450026f71306bef6acc150af4aa5ffe75801c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 03:12:58 +0200 Subject: [PATCH 088/433] autoconf: Detect boost, require version, set CXXFLAGS. This turns previous compiler errors complaining about missing files into proper ./configure time errors telling the user which version of boost is required. --- configure.ac | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/configure.ac b/configure.ac index 398bac047..426d3b674 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,13 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) + + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From 1f97b16b1d9d93e083a4f1436ba002e073cbe379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Mon, 1 Jul 2019 18:31:54 +0200 Subject: [PATCH 089/433] autoconf: Work around editline not being found on Ubuntu 16.04. And probably other Linux distributions with long-term support releases. Also update manual stating what version is needed; I checked that 1.14 is the oldest version with which current nix compiles, and added autoconf feature checks for some functions added in that release that nix uses. --- configure.ac | 11 ++++++++++- doc/manual/installation/prerequisites-source.xml | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 426d3b674..28be50575 100644 --- a/configure.ac +++ b/configure.ac @@ -171,7 +171,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) # Look for editline, a required dependency. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index e87d0de21..cbaa4d525 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -62,6 +62,10 @@ 1.66.0 or higher. It can be obtained from the official web site . + The editline library of version + 1.14.0 or higher. It can be obtained from the its repository + . + The xmllint and xsltproc programs to build this manual and the man-pages. These are part of the libxml2 and From 57daa860e8ed8432937aeecdcf6b9e952b0481b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Mon, 1 Jul 2019 19:04:03 +0200 Subject: [PATCH 090/433] autoconf: Fix C++17 detection not working on Ubuntu 16.04. And probably many other distributions. Until now, ./configure would fail silently printing a warning ./configure: line 4621: AX_CXX_COMPILE_STDCXX_17: command not found and then continuing, later failing with a C++ #error saying that some C++11 feature isn't supported (it didn't even get to the C++17 features). This is because older distributions don't come with the `AX_CXX_COMPILE_STDCXX_17` m4 macro. This commit vendors that macro accordingly. Now ./configure complains correctly: configure: error: *** A compiler with support for C++17 language features is required. On Ubuntu 16.04, ./configure completes if a newer compiler is used, e.g. with gcc-7 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test using: ./bootstrap.sh ./configure CXX=g++-7 --disable-doc-gen --with-boost=$(nix-build --no-link '' -A boost.dev) --- configure.ac | 3 +- .../installation/prerequisites-source.xml | 2 +- m4/ax_cxx_compile_stdcxx.m4 | 951 ++++++++++++++++++ m4/ax_cxx_compile_stdcxx_17.m4 | 35 + 4 files changed, 989 insertions(+), 2 deletions(-) create mode 100644 m4/ax_cxx_compile_stdcxx.m4 create mode 100644 m4/ax_cxx_compile_stdcxx_17.m4 diff --git a/configure.ac b/configure.ac index 28be50575..c12f9d5ff 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,5 @@ AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -62,7 +63,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_17 +AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) AC_CHECK_TOOL([AR], [ar]) diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index cbaa4d525..e7bdcf966 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -13,7 +13,7 @@ Bash Shell. The ./configure script relies on bashisms, so Bash is required. - A version of GCC or Clang that supports C++14. + A version of GCC or Clang that supports C++17. pkg-config to locate dependencies. If your distribution does not provide it, you can get diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 000000000..43087b2e6 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 000000000..a68341717 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016 Krzesimir Nowak +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) From fe068eca00d6a2dd2ea9ee08924dc3898c37fb6c Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 24 Jan 2017 22:42:15 +0000 Subject: [PATCH 091/433] mk: add support for passing LDFLAGS to libs and bins autotools-based systems usually allow user to append own LDFLAGS like LDFLAGS="-Wl,-O1 -Wl,--as-needed -Wl,--hash-style=gnu" at ./configure stage This change plumbs LDFLAGS through similar to existing CXXFLAGS variable. Signed-off-by: Sergei Trofimovich --- Makefile.config.in | 1 + mk/libraries.mk | 4 ++-- mk/programs.mk | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646..eabf1a426 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,6 +5,7 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_READLINE = @HAVE_READLINE@ diff --git a/mk/libraries.mk b/mk/libraries.mk index 3953446cb..307e29b9d 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -91,7 +91,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifneq ($(OS), Darwin) $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -105,7 +105,7 @@ define build-library $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifneq ($(OS), Darwin) diff --git a/mk/programs.mk b/mk/programs.mk index 2fbda12bd..d93df4468 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) @@ -46,7 +46,7 @@ define build-program _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) else From 20129bd83d57316cc0b69bec7abeec43011c56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 22:28:30 +0200 Subject: [PATCH 092/433] autoconf: Fix AC_STRUCT_DIRENT_D_TYPE being used before AC_PROG_CC. That was incorrect, because checking the dirent type already requires a working compiler. It had the effect that setting e.g. `: ${CFLAGS=""}` before `AC_PROG_CC` as per `AC_PROG_CC`'s documentation would have no effect, because `AC_STRUCT_DIRENT_D_TYPE` would automatically set CFLASGS. (In a followup commit `: ${CFLAGS=""}` will be used, so it's important to get this working first.) --- configure.ac | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index c12f9d5ff..3a486fab8 100644 --- a/configure.ac +++ b/configure.ac @@ -50,14 +50,6 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then - # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi - - CFLAGS= CXXFLAGS= AC_PROG_CC @@ -71,6 +63,14 @@ AC_CHECK_TOOL([AR], [ar]) AC_SYS_LARGEFILE +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) From 717e821b99797845e1bef47d862f8cb0fb69cfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 22:30:15 +0200 Subject: [PATCH 093/433] autoconf: Allow overriding CFLAGS/CXXFLAGS from outside. As is normal for autoconf-based projects. For example, it is a common use case to do ./configure CXXFLAGS=-O0 This did not work for nix until now, because the `CXXFLAGS=` declaration would unconditionally erase what the user had specified. The custom `OPTIMIZE` flag is removed, but the default `-O3` is retained; autoconf would default to `-g -O2` by default otherwise as documented on: https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C-Compiler.html https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C_002b_002b-Compiler.html --- Makefile | 7 ------- configure.ac | 6 ++++-- perl/Makefile | 7 ------- perl/configure.ac | 6 ++++-- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 45a3338ed..9ac82fda6 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/configure.ac b/configure.ac index 3a486fab8..571bb54d8 100644 --- a/configure.ac +++ b/configure.ac @@ -50,8 +50,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AC_PROG_CPP diff --git a/perl/Makefile b/perl/Makefile index 284c75022..f36f5d0e9 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index 966700695..e8e3610a8 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -2,8 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 From b49c3a9db516a87d38476966eff9994c13c5acf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 01:34:21 +0200 Subject: [PATCH 094/433] Makefile.config.in: Remove HAVE_READLINE. It was forgotten to be removed with commit c5f23f10a84f568874321c04984b1a14d2dce978 and so it until now stayed unsubstituted as `HAVE_READLINE = @HAVE_READLINE@` in Makefile.config. --- Makefile.config.in | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index eabf1a426..936a4f403 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,7 +8,6 @@ CXXFLAGS = @CXXFLAGS@ LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ -HAVE_READLINE = @HAVE_READLINE@ HAVE_SECCOMP = @HAVE_SECCOMP@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ From d203c554faa00fec55377d6640c8fb335a611a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 01:46:07 +0200 Subject: [PATCH 095/433] Fix C++ compatibility with older editline versions. For example, Ubuntu 16.04 and many similar long-term-support distros have older versions. --- src/nix/repl.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d8f812149..f857b2e89 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -9,7 +9,14 @@ #include #include #else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { #include +} #endif #include "shared.hh" From a96006d97fc87c5073f9a39db841625bdb7c401c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 02:47:27 +0200 Subject: [PATCH 096/433] Get BOOST_LDFLAGS from autoconf, fix Ubuntu 16.04 build. Our use of boost::coroutine2 depends on -lboost_context, which in turn depends on `-lboost_thread`, which in turn depends on `-lboost_system`. I suspect that this builds on nix only because of low-level hacks like NIX_LDFLAGS. This commit passes the proper linker flags, thus fixing bootstrap builds on non-nix distributions like Ubuntu 16.04. With these changes, I can build Nix on Ubuntu 16.04 using: ./bootstrap.sh ./configure --prefix=$HOME/editline-prefix \ --disable-doc-gen \ CXX=g++-7 \ --with-boost=$HOME/boost-prefix \ EDITLINE_CFLAGS=-I$HOME/editline-prefix/include \ EDITLINE_LIBS=-leditline \ LDFLAGS=-L$HOME/editline-prefix/lib make where * g++-7 comes from gcc-7 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test, * editline 1.14 from https://github.com/troglobit/editline/releases/tag/1.14.0 was installed into `$HOME/editline-prefix` (because Ubuntu 16.04's `editline` is too old to have the function nix uses), * boost 1.66 from https://www.boost.org/doc/libs/1_66_0/more/getting_started/unix-variants.html was installed into $HOME/boost-prefix (because Ubuntu 16.04 only has 1.58) --- Makefile.config.in | 1 + configure.ac | 3 +++ src/libutil/local.mk | 2 +- src/nix/local.mk | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 936a4f403..7e3b35b98 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,6 +9,7 @@ LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_SECCOMP = @HAVE_SECCOMP@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/configure.ac b/configure.ac index 571bb54d8..a52830b38 100644 --- a/configure.ac +++ b/configure.ac @@ -153,6 +153,9 @@ AC_SUBST(storedir) # and CPPFLAGS are not passed to the C++ compiler automatically. # Thus we append the returned CPPFLAGS to the CXXFLAGS here. AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" # Look for OpenSSL, a required dependency. diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 3ccc23fd5..e41a67d1f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,4 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d56..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -17,7 +17,7 @@ nix_SOURCES := \ nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From c3db9e6f8fd06d691be04cdd95a6bb21a400481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 03:34:17 +0200 Subject: [PATCH 097/433] autoconf: Check if --nonet works. Fixes #967 #506. Also give a helpful error message on what package the user likely has to install to make it work. --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index a52830b38..f510f8a97 100644 --- a/configure.ac +++ b/configure.ac @@ -258,6 +258,15 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) +# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) +if test "$doc_generate" = yes; then + AC_MSG_CHECKING([if docbook can run without networking]) + if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then + AC_MSG_RESULT(yes) + else + AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?]) + fi +fi # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) From cd8bc06e8786018ddb16cea4cb10971b63d0efd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 04:20:37 +0200 Subject: [PATCH 098/433] autoconf: Add comment on use of `false`. This is to avoid confusion as in commit a0d29040f79b365598fe75d01f72d29ab538206b. --- configure.ac | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index f510f8a97..1e3a8091c 100644 --- a/configure.ac +++ b/configure.ac @@ -119,10 +119,12 @@ if test -z "$$1"; then fi ]) +# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program +# is not required in all cases. (e.g. not when building from a release tarball). +# The use of `false` isn't ideal, because when used somewhere in the build it +# will silently fail without error message; this should be improved. NEED_PROG(bash, bash) NEED_PROG(patch, patch) -AC_PATH_PROG(xmllint, xmllint, false) -AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) NEED_PROG(sed, sed) @@ -258,8 +260,12 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) if test "$doc_generate" = yes; then + # Programs required to build the manual + NEED_PROG(xmllint, xmllint) + NEED_PROG(xsltproc, xsltproc) + + # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) AC_MSG_CHECKING([if docbook can run without networking]) if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then AC_MSG_RESULT(yes) From 82b7f0e840983879a510245903ff7c917276f65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 04:28:06 +0200 Subject: [PATCH 099/433] autoconf: Implement release tarball detection. Fixes #257. This should finally allow us to address all cases of build errors due to differences between release tarballs and building from git. See also https://github.com/NixOS/nix/issues/506#issuecomment-507312587 --- configure.ac | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 1e3a8091c..8bc14d476 100644 --- a/configure.ac +++ b/configure.ac @@ -5,6 +5,19 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED +# Detect building from release tarballs (we need less dependencies then) +# by checking for the `.dist-files` file present only in release tarballs +# (created in release.nix). +# For example, we don't need to build the manual or run bison/flex +# parser generators for release tarballs, as those have them pre-generated. +AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)]) +if test -f .dist-files; then + AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs) + building_from_release_tarball=yes; +else + AC_MSG_RESULT(no) +fi + # Construct a Nix system name (like "i686-linux"). AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -119,14 +132,13 @@ if test -z "$$1"; then fi ]) -# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program -# is not required in all cases. (e.g. not when building from a release tarball). -# The use of `false` isn't ideal, because when used somewhere in the build it -# will silently fail without error message; this should be improved. NEED_PROG(bash, bash) NEED_PROG(patch, patch) -AC_PATH_PROG(flex, flex, false) -AC_PATH_PROG(bison, bison, false) +if test "x$building_from_release_tarball" != x"yes"; then + # Parser generators are pre-built in release tarballs. + NEED_PROG(flex, flex) + NEED_PROG(bison, bison) +fi NEED_PROG(sed, sed) NEED_PROG(tar, tar) NEED_PROG(bzip2, bzip2) @@ -260,7 +272,8 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -if test "$doc_generate" = yes; then +# The manual is pre-generated in release tarballs. +if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then # Programs required to build the manual NEED_PROG(xmllint, xmllint) NEED_PROG(xsltproc, xsltproc) From e486d8d40e626a20e06d792db8cc5ac5aba9a5b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 00:33:51 +0200 Subject: [PATCH 100/433] Revert 82b7f0e840983879a510245903ff7c917276f65d, cd8bc06e8786018ddb16cea4cb10971b63d0efd2, c3db9e6f8fd06d691be04cdd95a6bb21a400481d This breaks the tarball job: https://hydra.nixos.org/build/95714570 --- configure.ac | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/configure.ac b/configure.ac index 8bc14d476..a52830b38 100644 --- a/configure.ac +++ b/configure.ac @@ -5,19 +5,6 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Detect building from release tarballs (we need less dependencies then) -# by checking for the `.dist-files` file present only in release tarballs -# (created in release.nix). -# For example, we don't need to build the manual or run bison/flex -# parser generators for release tarballs, as those have them pre-generated. -AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)]) -if test -f .dist-files; then - AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs) - building_from_release_tarball=yes; -else - AC_MSG_RESULT(no) -fi - # Construct a Nix system name (like "i686-linux"). AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -134,11 +121,10 @@ fi NEED_PROG(bash, bash) NEED_PROG(patch, patch) -if test "x$building_from_release_tarball" != x"yes"; then - # Parser generators are pre-built in release tarballs. - NEED_PROG(flex, flex) - NEED_PROG(bison, bison) -fi +AC_PATH_PROG(xmllint, xmllint, false) +AC_PATH_PROG(xsltproc, xsltproc, false) +AC_PATH_PROG(flex, flex, false) +AC_PATH_PROG(bison, bison, false) NEED_PROG(sed, sed) NEED_PROG(tar, tar) NEED_PROG(bzip2, bzip2) @@ -272,20 +258,6 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -# The manual is pre-generated in release tarballs. -if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then - # Programs required to build the manual - NEED_PROG(xmllint, xmllint) - NEED_PROG(xsltproc, xsltproc) - - # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) - AC_MSG_CHECKING([if docbook can run without networking]) - if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then - AC_MSG_RESULT(yes) - else - AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?]) - fi -fi # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) From 648bdf153d2e4d3c6687d8e1780b0dfd0aa98cda Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 21 May 2019 22:29:23 -0400 Subject: [PATCH 101/433] tarball-ttl: document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incorporates text from Niklas Hambüchen in #2978 Closes #1115 --- doc/manual/command-ref/conf-file.xml | 25 +++++++++++++++++++++++++ doc/manual/expressions/builtins.xml | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 24fbf28cf..09aad2e05 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -864,6 +864,31 @@ requiredSystemFeatures = [ "kvm" ]; + tarball-ttl + + + Default: 3600 seconds. + + The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + + + Setting the TTL to 0 forces Nix to always + check if the tarball is up to date. + + Nix caches tarballs in + $XDG_CACHE_HOME/nix/tarballs. + + Files fetched via NIX_PATH, + fetchGit, fetchMercurial, + fetchTarball, and fetchurl + respect this TTL. + + + timeout diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index a87639a07..deffcb594 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -347,7 +347,7 @@ stdenv.mkDerivation { … } You can change the cache timeout either on the command line with or in the Nix configuration file with this option: - tarball-ttl number of seconds to cache. + number of seconds to cache. Note that when obtaining the hash with nix-prefetch-url @@ -498,7 +498,7 @@ stdenv.mkDerivation { … } fetch the latest version of a remote branch. Nix will refetch the branch in accordance to - . + . This behavior is disabled in Pure evaluation mode. builtins.fetchGit { From aa739e783993864aa6e0c8a4820e6b59f4626d92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 10:29:41 +0200 Subject: [PATCH 102/433] nix copy: Rename --substitute to --substitute-on-destination '--substitute' was being shadowed by the regular '--substitute' (the short-hand for '--option substitute true'). Fixes #2983. --- src/nix/copy.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 96bd453d8..12a9f9cd3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -36,7 +36,7 @@ struct CmdCopy : StorePathsCommand .set(&checkSigs, NoCheckSigs); mkFlag() - .longName("substitute") + .longName("substitute-on-destination") .shortName('s') .description("whether to try substitutes on the destination store (only supported by SSH)") .set(&substitute, Substitute); From 03f09e1d18c39706b917964f14f9e40513e4aaea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 19:46:15 +0200 Subject: [PATCH 103/433] Revert "Fix 'error 9 while decompressing xz file'" This reverts commit 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91. --- src/libstore/binary-cache-store.cc | 18 +++--- src/libstore/download.cc | 67 ++++++++++++++++---- src/libstore/download.hh | 11 +--- src/libstore/http-binary-cache-store.cc | 55 +++++----------- src/libstore/store-api.cc | 84 ++++++++++++------------- src/libutil/retry.hh | 38 ----------- src/libutil/types.hh | 2 - 7 files changed, 119 insertions(+), 156 deletions(-) delete mode 100644 src/libutil/retry.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8b736056e..4527ee6ba 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,8 +10,6 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" -#include "retry.hh" -#include "download.hh" #include @@ -81,15 +79,13 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; - }); + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 7a2af237e..2267b5d35 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,7 +8,6 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" -#include "retry.hh" #ifdef ENABLE_S3 #include @@ -20,9 +19,11 @@ #include #include +#include #include #include #include +#include #include using namespace std::string_literals; @@ -45,6 +46,9 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; + std::random_device rd; + std::mt19937 mt19937; + struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -57,6 +61,12 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + struct curl_slist * requestHeaders = 0; std::string encoding; @@ -377,7 +387,9 @@ struct CurlDownloader : public Downloader } } - fail( + attempt++; + + auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -388,15 +400,31 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code))); + request.verb(), request.uri, curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. */ + if (err == Transient && attempt < request.tries) { + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + downloader.enqueueItem(shared_from_this()); + } + else + fail(exc); } } }; struct State { + struct EmbargoComparator { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + return i1->embargo > i2->embargo; + } + }; bool quit = false; - std::vector> incoming; + std::priority_queue, std::vector>, EmbargoComparator> incoming; }; Sync state_; @@ -409,6 +437,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() + : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -502,7 +531,9 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue. */ + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -511,9 +542,22 @@ struct CurlDownloader : public Downloader } std::vector> incoming; + auto now = std::chrono::steady_clock::now(); + { auto state(state_.lock()); - std::swap(state->incoming, incoming); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() + || item->embargo < nextWakeup) + nextWakeup = item->embargo; + break; + } + } quit = state->quit; } @@ -540,7 +584,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - state->incoming.clear(); + while (!state->incoming.empty()) state->incoming.pop(); state->quit = true; } } @@ -556,7 +600,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push_back(item); + state->incoming.push(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -639,9 +683,7 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return retry(request.tries, [&]() { - return enqueueDownload(request).get(); - }); + return enqueueDownload(request).get(); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -827,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - warn("%s; using cached result", e.msg()); + printError(format("warning: %1%; using cached result") % e.msg()); result.etag = expectedETag; } } @@ -878,4 +920,5 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } + } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 9e965b506..3b7fff3ba 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -96,13 +96,11 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. The request will be retried in - case of transient failures. */ + /* Synchronously download a file. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. The request will not be - retried in case of transient failures. */ + invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -128,11 +126,6 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } - - bool isTransient() override - { - return error == Downloader::Error::Transient; - } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5633b4355..3d17a8966 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,7 +2,6 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" -#include "retry.hh" namespace nix { @@ -114,6 +113,7 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); + request.tries = 8; return request; } @@ -136,46 +136,21 @@ protected: { checkEnabled(); - struct State - { - DownloadRequest request; - std::function tryDownload; - unsigned int attempt = 0; - State(DownloadRequest && request) : request(request) {} - }; + auto request(makeRequest(path)); - auto state = std::make_shared(makeRequest(path)); - - state->tryDownload = [callback, state, this]() { - getDownloader()->enqueueDownload(state->request, - {[callback, state, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - ++state->attempt; - if (state->attempt < state->request.tries && e.isTransient()) { - auto ms = retrySleepTime(state->attempt); - warn("%s; retrying in %d ms", e.what(), ms); - /* We can't sleep here because that would - block the download thread. So use a - separate thread for sleeping. */ - std::thread([state, ms]() { - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); - state->tryDownload(); - }).detach(); - } else { - maybeDisable(); - callback.rethrow(); - } - } catch (...) { - callback.rethrow(); - } - }}); - }; - - state->tryDownload(); + getDownloader()->enqueueDownload(request, + {[callback, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + maybeDisable(); + callback.rethrow(); + } catch (...) { + callback.rethrow(); + } + }}); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 92f01fd2e..f5608d384 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,11 +6,10 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" -#include "retry.hh" -#include "download.hh" #include + namespace nix { @@ -580,57 +579,54 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - retry(downloadSettings.tries, [&]() { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); + auto info = srcStore->queryPathInfo(storePath); - auto info = srcStore->queryPathInfo(storePath); + uint64_t total = 0; - uint64_t total = 0; + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } - - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); - }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); }); - - dstStore->addToStore(*info, *source, repair, checkSigs); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); + + dstStore->addToStore(*info, *source, repair, checkSigs); } diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh deleted file mode 100644 index b45cb37f7..000000000 --- a/src/libutil/retry.hh +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "logging.hh" - -#include -#include -#include -#include - -namespace nix { - -inline unsigned int retrySleepTime(unsigned int attempt) -{ - std::random_device rd; - std::mt19937 mt19937; - return 250.0 * std::pow(2.0f, - attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); -} - -template -C retry(unsigned int attempts, std::function && f) -{ - unsigned int attempt = 0; - while (true) { - try { - return f(); - } catch (BaseError & e) { - ++attempt; - if (attempt >= attempts || !e.isTransient()) - throw; - auto ms = retrySleepTime(attempt); - warn("%s; retrying in %d ms", e.what(), ms); - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); - } - } -} - -} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 88e3243f4..92bf469b5 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,8 +109,6 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); - - virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ From f76b2a7fdd7dabad7891400248f5cc778182ebf3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 22:27:50 +0200 Subject: [PATCH 104/433] Downloader: Use warn() --- src/libstore/download.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 2267b5d35..195eccc2a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -406,7 +406,7 @@ struct CurlDownloader : public Downloader download after a while. */ if (err == Transient && attempt < request.tries) { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } @@ -579,7 +579,7 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError(format("unexpected error in download thread: %s") % e.what()); + printError("unexpected error in download thread: %s", e.what()); } { @@ -869,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("warning: %s; using cached result", e.msg()); result.etag = expectedETag; } } From 00f6fafad61db4537268e1ffa636fd0d2ae86059 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 23:05:04 +0200 Subject: [PATCH 105/433] HttpBinaryCacheStore: Use default number of retries for NARs --- src/libstore/http-binary-cache-store.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3d17a8966..df2fb9332 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -113,7 +113,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } From 53247d6b116905e7233b1efd6c14845e20d27442 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 23:12:17 +0200 Subject: [PATCH 106/433] Resume NAR downloads This is a much simpler fix to the 'error 9 while decompressing xz file' problem than 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91. We just do a ranged HTTP request starting after the data that we previously wrote into the sink. Fixes #2952, #379. --- src/libstore/download.cc | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 195eccc2a..91087eebc 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -71,6 +71,10 @@ struct CurlDownloader : public Downloader std::string encoding; + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, Callback callback) @@ -81,9 +85,10 @@ struct CurlDownloader : public Downloader {request.uri}, request.parentAct) , callback(callback) , finalSink([this](const unsigned char * data, size_t len) { - if (this->request.dataCallback) + if (this->request.dataCallback) { + writtenToSink += len; this->request.dataCallback((char *) data, len); - else + } else this->result.data->append((char *) data, len); }) { @@ -161,6 +166,7 @@ struct CurlDownloader : public Downloader status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); result.bodySize = 0; + acceptRanges = false; encoding = ""; } else { auto i = line.find(':'); @@ -178,7 +184,9 @@ struct CurlDownloader : public Downloader return 0; } } else if (name == "content-encoding") - encoding = trim(string(line, i + 1));; + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; } } return realSize; @@ -296,6 +304,9 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + result.data = std::make_shared(); result.bodySize = 0; } @@ -330,7 +341,7 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; done = true; @@ -403,10 +414,20 @@ struct CurlDownloader : public Downloader request.verb(), request.uri, curl_easy_strerror(code), code)); /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - warn("%s; retrying in %d ms", exc.what(), ms); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } From 5e0a64229b3d244d46b3b5f9fae964d01958c15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 18 Jul 2019 10:57:26 +0200 Subject: [PATCH 107/433] Add Open Collective --- .github/FUNDING.yml | 2 +- README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index bbaabf93c..15792e713 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -custom: https://nixos.org/nixos/foundation.html +open_collective: nixos diff --git a/README.md b/README.md index 3173c6c44..48cb1685c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) + Nix, the purely functional package manager ------------------------------------------ From cf6172f05e9446b2b85dd7fa2e73cb93f56620e2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 28 Mar 2019 16:04:02 -0400 Subject: [PATCH 108/433] docs: document balancing cores and max-jobs --- .../advanced-topics/advanced-topics.xml | 1 + doc/manual/advanced-topics/cores-vs-jobs.xml | 121 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 8 +- 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 doc/manual/advanced-topics/cores-vs-jobs.xml diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..12a826254 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,6 +7,7 @@ Advanced Topics + diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000..eba645faf --- /dev/null +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ + + +Tuning Cores and Jobs + +Nix has two relevant settings with regards to how your CPU cores +will be utilized: and +. This chapter will talk about what +they are, how they interact, and their configuration trade-offs. + + + + + + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + + + + + + Suggests how many cores each derivation should use. Similar to + make -j. + + + + +The setting determines the value of +NIX_BUILD_CORES. NIX_BUILD_CORES is equal +to , unless +equals 0, in which case NIX_BUILD_CORES +will be the total number of cores in the system. + +The total number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. + +The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Balancing 24 Build Cores
NIX_BUILD_CORESMaximum ProcessesResult
1242424 + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. +
46624 + Four derivations will be built at once, each given access to + six cores. +
126672 + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. +
241124 + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. +
24024576 + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. +
+ +It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +NIX_BUILD_CORES environment variable. + + diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 09aad2e05..4f7393e2e 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -238,8 +238,9 @@ false. linkend='opt-cores'>--cores command line switch and defaults to 1. The value 0 means that the builder should use all available CPU cores in the - system. + system. + See also . diff-hook @@ -498,7 +499,10 @@ builtins.fetchurl { regardless). It can be overridden using the () - command line switch. + command line switch. + + See also . + max-silent-time From 9031a6838ccf10b0db0d91a89e14d71a1adf6235 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Tue, 23 Jul 2019 15:21:09 +0200 Subject: [PATCH 109/433] Remove .github/FUNDING.yml The configuration is now done through the shared configuration repo: https://github.com/nixos/.github --- .github/FUNDING.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 15792e713..000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,3 +0,0 @@ -# These are supported funding model platforms - -open_collective: nixos From c82a856b36bcd008edeb9c4c981b974ae642afea Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:39:44 -0400 Subject: [PATCH 110/433] Add default for USER when unset uses $(id -u -n) when USER is unset, this is needed on some weird setups in Docker. Fixes #971 --- scripts/install-nix-from-closure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index fc999d336..35926f3da 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 fi -if [ -z "$USER" ]; then +if [ -z "$USER" ] && ! USER=$(id -u -n); then echo "$0: \$USER is not set" >&2 exit 1 fi From 03addc3b0a79cb7a58ae5651d1915af7383a220c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:43:24 -0400 Subject: [PATCH 111/433] Use $HOME instead of $USER $USER/.nix-profile will not be a path. I think $HOME/.nix-profile was the origininal intent. /cc @Grahamc --- tests/install-darwin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh index c99ce84ac..9933eba94 100755 --- a/tests/install-darwin.sh +++ b/tests/install-darwin.sh @@ -34,7 +34,7 @@ cleanup() { sudo rm -rf /etc/nix \ /nix \ /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ - "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels" + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" } verify() { From d171090530f4a2a79efec2c385bee1a10844c706 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:37:57 -0400 Subject: [PATCH 112/433] =?UTF-8?q?Disable=20CLONE=5FNEWUSER=20when=20it?= =?UTF-8?q?=E2=80=99s=20unavailable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some kernels disable "unpriveleged user namespaces". This is unfortunate, but we can still use mount namespaces. Anyway, since each builder has its own nixbld user, we already have most of the benefits of user namespaces. --- src/libstore/build.cc | 14 ++++++++++++-- src/nix/run.cc | 5 ++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..c10005839 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2302,10 +2302,20 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345..90b76d666 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -199,7 +199,10 @@ void chrootHelper(int argc, char * * argv) uid_t gid = getgid(); if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot From 11d853462925d0b57fe956962e07edf5751fd4c3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 14:29:58 -0400 Subject: [PATCH 113/433] Use sandbox fallback when cloning fails in builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When sandbox-fallback = true (the default), the Nix builder will fall back to disabled sandbox mode when the kernel doesn’t allow users to set it up. This prevents hard errors from occuring in tricky places, especially the initial installer. To restore the previous behavior, users can set: sandbox-fallback = false in their /etc/nix/nix.conf configuration. --- src/libstore/build.cc | 12 +++++++++++- src/libstore/globals.hh | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c10005839..dd08ce7d7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2316,13 +2316,22 @@ void DerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(EPERM); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res == EPERM && settings.sandboxFallback) { + useChroot = false; + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2353,6 +2362,7 @@ void DerivationGoal::startBuilder() } else #endif { + fallback: options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..cc9534b27 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -209,6 +209,9 @@ public: "The paths to make available inside the build sandbox.", {"build-chroot-dirs", "build-sandbox-paths"}}; + Setting sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting extraSandboxPaths{this, {}, "extra-sandbox-paths", "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; From cd933b22d2041b7efc348dcc09ff255967ffc663 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Sat, 27 Jul 2019 19:40:19 -0700 Subject: [PATCH 114/433] Add pname and version to nix-env -q --json --- src/nix-env/nix-env.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 56ed75dae..87b2e43f0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -860,7 +860,10 @@ static void queryJSON(Globals & globals, vector & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.queryName()); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); @@ -1026,10 +1029,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.queryName(); - else if (printName) + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the From 89865144c3ba0162cd37bcbe49b3095e9bab4164 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 30 Jul 2019 11:22:55 +0200 Subject: [PATCH 115/433] Allow builtins.pathExists to check the existence of /nix/store paths This makes it consitent with builtins.readDir. --- src/libexpr/primops.cc | 10 ++++++++-- tests/import-derivation.nix | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 070e72f3a..350dba474 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -832,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix index 91adcd288..44fa9a45d 100644 --- a/tests/import-derivation.nix +++ b/tests/import-derivation.nix @@ -10,7 +10,10 @@ let ''; }; - value = import bar; + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; in From ee1e3132cab9d7de14837f2b04b115e544ae1622 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 30 Jul 2019 11:29:03 +0200 Subject: [PATCH 116/433] Disable findRuntimeRoots on darwin when running tests because lsof is slow See: https://github.com/NixOS/nix/issues/3011 --- src/libstore/gc.cc | 25 +++++++++++++++---------- tests/common.sh.in | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26e2b0dca..83cdf13a6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -444,17 +444,22 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #if !defined(__linux__) - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ } #endif diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..15d7b1ef9 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -16,6 +16,7 @@ if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home From 9a0855bbb6546e792848e551e79f8efc40782eeb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 30 Jul 2019 17:52:42 -0400 Subject: [PATCH 117/433] =?UTF-8?q?Don=E2=80=99t=20rely=20on=20EPERM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit startProcess does not appear to send the exit code to the helper correctly. Not sure why this is, but it is probably safe to just fallback on all sandbox errors. --- src/libstore/build.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dd08ce7d7..0f71e7511 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2320,7 +2320,7 @@ void DerivationGoal::startBuilder() parent. This is only done when sandbox-fallback is set to true (the default). */ if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(EPERM); + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); @@ -2328,7 +2328,7 @@ void DerivationGoal::startBuilder() }, options); int res = helper.wait(); - if (res == EPERM && settings.sandboxFallback) { + if (res != 0 && settings.sandboxFallback) { useChroot = false; goto fallback; } else if (res != 0) From 320126aeebc89f8c34ee1668ebacf1fe4b512ed8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 14:04:09 +0200 Subject: [PATCH 118/433] Tweak min-free/max-free descriptions --- doc/manual/command-ref/conf-file.xml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 4f7393e2e..0a97719ef 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -483,8 +483,10 @@ builtins.fetchurl { max-free - This option defines after how many free bytes to stop collecting - garbage once the min-free condition gets triggered. + When a garbage collection is triggered by the + min-free option, it stops as soon as + max-free bytes are available. The default is + infinity (i.e. delete all garbage). @@ -528,9 +530,11 @@ builtins.fetchurl { min-free - When the disk reaches min-free bytes of free disk space during a build, nix - will start to garbage-collection until max-free bytes are available on the disk. - A value of 0 (the default) means that this feature is disabled. + When free disk space in /nix/store + drops below min-free during a build, Nix + performs a garbage-collection until max-free + bytes are available or there is no more garbage. A value of + 0 (the default) disables this feature. From 7c5596734f65b30b250ea73a423b40a4ce513fdf Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 11 Jul 2019 20:23:03 +0200 Subject: [PATCH 119/433] Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen Co-authored-by: Eelco Dolstra --- .../advanced-topics/advanced-topics.xml | 1 + .../advanced-topics/post-build-hook.xml | 157 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 55 ++++++ src/libstore/build.cc | 55 ++++++ src/libstore/globals.hh | 3 + src/libutil/logging.hh | 2 + src/libutil/util.cc | 23 ++- src/libutil/util.hh | 2 + src/nix/progress-bar.cc | 16 +- tests/dependencies.nix | 1 + tests/local.mk | 3 +- tests/post-hook.sh | 15 ++ tests/push_to_store.sh | 4 + 13 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 doc/manual/advanced-topics/post-build-hook.xml create mode 100644 tests/post-hook.sh create mode 100755 tests/push_to_store.sh diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..1b8841ad2 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -8,5 +8,6 @@ + diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..4335b308b --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,157 @@ + + +Using the <xref linkend="conf-post-build-hook" /> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +we will restart the Nix daemon a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will only contain characters which are safe for word + splitting, and free of any globs. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 09aad2e05..d2c9c7502 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -656,6 +656,61 @@ password my-password + + post-build-hook + + Optional. The path to a program to execute after each build. + + This option is only settable in the global + nix.conf, or on the command line by trusted + users. + + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. + + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + + + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space ( ) character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + repeat How many times to repeat builds to check whether diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..7494eec41 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..7dea68921 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -315,6 +315,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f2219445..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -26,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -36,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5c..44fa72482 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -84,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8d..b538a0b41 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -270,12 +270,14 @@ struct RunOptions std::optional uid; std::optional gid; std::optional chdir; + std::optional> environment; Path program; bool searchPath = true; Strings args; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index b1c1d87de..c0bcfb0c9 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -170,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -228,14 +236,18 @@ public: update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add..eca4b2964 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..8daaa859f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -26,7 +26,8 @@ nix_tests = \ check.sh \ plugins.sh \ search.sh \ - nix-copy-ssh.sh + nix-copy-ssh.sh \ + post-hook.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 000000000..3f82d0c5e --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push_to_store.sh b/tests/push_to_store.sh new file mode 100755 index 000000000..d97eb095d --- /dev/null +++ b/tests/push_to_store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs From ec415d7166d607c92cf8f1af688f86e4b4731dff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 17:07:33 +0200 Subject: [PATCH 120/433] Add a test for auto-GC This currently fails because we're using POSIX file locks. So when the garbage collector opens and closes its own temproots file, it causes the lock to be released and then deleted by another GC instance. --- src/libstore/gc.cc | 9 +++++-- src/libstore/globals.hh | 3 +++ tests/gc-auto.sh | 59 +++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 4 ++- 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 tests/gc-auto.sh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 83cdf13a6..eeb237839 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -871,7 +871,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -892,7 +897,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..0c0a9ec54 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -342,6 +342,9 @@ public: Setting maxFree{this, std::numeric_limits::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 000000000..1e91282d0 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,59 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) +garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) +garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +expr2=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..261e4db0d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,9 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - gc.sh gc-concurrent.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ From e349f2c0a370e4dfd09ae51c2cae4db08a834ad5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Oct 2018 15:03:03 +0200 Subject: [PATCH 121/433] Use BSD instead of POSIX file locks POSIX file locks are essentially incompatible with multithreading. BSD locks have much saner semantics. We need this now that there can be multiple concurrent LocalStore::buildPaths() invocations. --- src/libstore/build.cc | 17 ----- src/libstore/gc.cc | 4 +- src/libstore/local-store.hh | 2 +- src/libstore/pathlocks.cc | 125 ++++++++++++------------------------ src/libstore/pathlocks.hh | 4 -- 5 files changed, 45 insertions(+), 107 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..a28619ab9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3984,17 +3984,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); @@ -4034,12 +4023,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index eeb237839..2c791efbe 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6b655647b..af8b84bf5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -263,7 +263,7 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..b6d8547c7 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -7,7 +7,7 @@ #include #include -#include +#include namespace nix { @@ -40,17 +40,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +55,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +67,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +80,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +88,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } + AutoCloseFD fd; - try { + while (1) { - AutoCloseFD fd; + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); + debug(format("lock acquired on '%1%'") % lockPath); - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +157,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +174,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } From a2597d5f27bc6cfa26343be2f481c8a9d1e22753 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 18:37:55 +0200 Subject: [PATCH 122/433] Simplify With BSD locks we don't have to guard against reading our own temproots. --- src/libstore/gc.cc | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 2c791efbe..366dbfb0a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); From 399b6f3c46077e10a7047e8216fc1a67425a768a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Oct 2018 14:46:30 +0200 Subject: [PATCH 123/433] nix-store --verify: Don't repair while holding the GC lock --- src/libstore/local-store.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..63b11467e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1210,7 +1210,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1221,13 +1222,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); From 363a2f68261af73aefe4edead9c0526030751a27 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 6 Aug 2019 14:26:43 -0400 Subject: [PATCH 124/433] post-build-hook: docs fixup --- doc/manual/advanced-topics/post-build-hook.xml | 7 +++++-- doc/manual/command-ref/conf-file.xml | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 4335b308b..3dc43ee79 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -74,6 +74,8 @@ trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDS #!/bin/sh set -eu +set -f # disable globbing +export IFS=' ' echo "Signing paths" $OUT_PATHS nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS @@ -88,8 +90,9 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS list of Nix store paths. In this case, we expect and want the shell to perform word splitting to make each output path its own argument to nix sign-paths. Nix guarantees - the paths will only contain characters which are safe for word - splitting, and free of any globs. + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index d2c9c7502..e818a74cd 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -674,6 +674,7 @@ password my-password The hook does not execute on substituted paths. The hook's output always goes to the user's terminal. If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. The program executes with no arguments. The program's environment @@ -693,7 +694,7 @@ password my-password OUT_PATHS - Output paths of the built derivation, separated by a space ( ) character. + Output paths of the built derivation, separated by a space character. Example: /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc From c3fefd1a6efec457395a187d15f435447dee6f3b Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 7 Aug 2019 07:34:11 -0500 Subject: [PATCH 125/433] pathlocks: add include to fcntl.h for O_CLOEXEC --- src/libstore/pathlocks.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index b6d8547c7..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include From 1eeaf99cf89630f84b4a1b6882a0f98019725d9e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 7 Aug 2019 14:45:07 -0400 Subject: [PATCH 126/433] fixup: docs for post-build-hook --- doc/manual/command-ref/conf-file.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index f3c721c78..c7b540892 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -682,7 +682,7 @@ password my-password The hook does not execute on substituted paths. The hook's output always goes to the user's terminal. If the hook fails, the build succeeds but no further builds execute. - The hook executes synchronously, and blocks other builds from progressing while it runs. + The hook executes synchronously, and blocks other builds from progressing while it runs. The program executes with no arguments. The program's environment From 2053ac774797f237a5335c4232bc1ed8902686cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Aug 2019 12:18:46 +0200 Subject: [PATCH 127/433] Rename file for consistency --- tests/post-hook.sh | 2 +- tests/{push_to_store.sh => push-to-store.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{push_to_store.sh => push-to-store.sh} (100%) diff --git a/tests/post-hook.sh b/tests/post-hook.sh index 3f82d0c5e..54d605d63 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -5,7 +5,7 @@ clearStore export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store -nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh +nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh clearStore diff --git a/tests/push_to_store.sh b/tests/push-to-store.sh similarity index 100% rename from tests/push_to_store.sh rename to tests/push-to-store.sh From 05a10dd835923092b54fcad2276d54bf164c1b7f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Aug 2019 14:39:05 +0200 Subject: [PATCH 128/433] tests/post-hook.sh: Don't put result link in cwd --- tests/post-hook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/post-hook.sh b/tests/post-hook.sh index 54d605d63..a02657215 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -5,7 +5,7 @@ clearStore export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store -nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh clearStore From a02457db715a80de7630d823fd8d7230a4bd451f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 7 Aug 2019 15:14:40 -0400 Subject: [PATCH 129/433] conf: stalled-download-timeout: make tunable Make curl's low speed limit configurable via stalled-download-timeout. Before, this limit was five minutes without receiving a single byte. This is much too long as if the remote end may not have even acknowledged the HTTP request. --- doc/manual/command-ref/conf-file.xml | 8 ++++++++ src/libstore/download.cc | 4 +--- src/libstore/download.hh | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c7b540892..1325a3f22 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -873,6 +873,14 @@ password my-password + stalled-download-timeout + + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration. + + + substituters A list of URLs of substituters, separated by diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 91087eebc..c322d267d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 3b7fff3ba..c68381846 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -24,6 +24,9 @@ struct DownloadSettings : Config Setting connectTimeout{this, 0, "connect-timeout", "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + Setting stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; }; From ee9c988a1b2e3c511b8613e698a0f9632ab1538f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 12 Apr 2019 12:31:33 -0400 Subject: [PATCH 130/433] Track function start and ends for flame graphs With this patch, and this file I called `log.py`: #!/usr/bin/env nix-shell #!nix-shell -i python3 -p python3 --pure import sys from pprint import pprint stack = [] timestack = [] for line in open(sys.argv[1]): components = line.strip().split(" ", 2) if components[0] != "function-trace": continue direction = components[1] components = components[2].rsplit(" ", 2) loc = components[0] _at = components[1] time = int(components[2]) if direction == "entered": stack.append(loc) timestack.append(time) elif direction == "exited": dur = time - timestack.pop() vst = ";".join(stack) print(f"{vst} {dur}") stack.pop() and: nix-instantiate --trace-function-calls -vvvv ../nixpkgs/pkgs/top-level/release.nix -A unstable > log.matthewbauer 2>&1 ./log.py ./log.matthewbauer > log.matthewbauer.folded flamegraph.pl --title matthewbauer-post-pr log.matthewbauer.folded > log.matthewbauer.folded.svg I can make flame graphs like: http://gsc.io/log.matthewbauer.folded.svg --- Includes test cases around function call failures and tryEval. Uses RAII so the finish is always called at the end of the function. --- contrib/stack-collapse.py | 39 +++++++++++++ doc/manual/command-ref/conf-file.xml | 28 +++++++++ src/libexpr/eval.cc | 8 ++- src/libexpr/eval.hh | 4 ++ src/libexpr/function-trace.hh | 24 ++++++++ tests/function-trace.sh | 86 ++++++++++++++++++++++++++++ tests/local.mk | 3 +- 7 files changed, 189 insertions(+), 3 deletions(-) create mode 100755 contrib/stack-collapse.py create mode 100644 src/libexpr/function-trace.hh create mode 100755 tests/function-trace.sh diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 000000000..ee77e1589 --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `-vvvv` and +# `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls -vvvv '' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c7b540892..407e5d10b 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -973,6 +973,34 @@ requiredSystemFeatures = [ "kvm" ]; + trace-function-calls + + + + Default: false. + + If set to true, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit. + + +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + + The undefined position means the function + call is a builtin. + + Use the contrib/stack-collapse.py script + distributed with the Nix source code to convert the trace logs + in to a format suitable for flamegraph.pl. + + + + + trusted-public-keys A whitespace-separated list of public keys. When diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..9f4b6b411 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "json.hh" #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include #include #if HAVE_BOEHMGC @@ -1094,9 +1094,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..22472fd72 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include #include @@ -349,6 +350,9 @@ struct EvalSettings : Config Setting allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8234b7603 --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + vomit("function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + vomit("function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 000000000..f7d93b435 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,86 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + -vvvv \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/local.mk b/tests/local.mk index ef359c631..187f96ea2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -29,7 +29,8 @@ nix_tests = \ plugins.sh \ search.sh \ nix-copy-ssh.sh \ - post-hook.sh + post-hook.sh \ + function-trace.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 653c40778462b5921fbbe2abffd1a2822f08ec57 Mon Sep 17 00:00:00 2001 From: Daniel Diaz Date: Fri, 14 Jun 2019 00:05:31 +0200 Subject: [PATCH 131/433] Expanded documentation for .nix-defexpr --- doc/manual/command-ref/nix-env.xml | 40 +++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 56c466268..693f23f7f 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -221,31 +221,53 @@ also . ~/.nix-defexpr - A directory that contains the default Nix + The source for the default Nix expressions used by the , , and operations to obtain derivations. The + --available operations to obtain derivations. The option may be used to override this default. - The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if ~/.nix-defexpr contains - two files, foo and bar, + If ~/.nix-defexpr is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression. + + If ~/.nix-defexpr is a directory + containing a default.nix file, that file + is loaded as in the above paragraph. + + If ~/.nix-defexpr is a directory without + a default.nix file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + + + For example, if ~/.nix-defexpr contains + two files, foo.nix and bar.nix, then the default Nix expression will essentially be { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; } + The file manifest.nix is always ignored. + Subdirectories without a default.nix file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression. + The command nix-channel places symlinks to the downloaded Nix expressions from each subscribed channel in this directory. - From 91b00b145f0c50e346d0250168cbcbcba7aa3b40 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 12 Aug 2019 21:03:48 +0200 Subject: [PATCH 132/433] libutil: add SizedSource Introduce the SizeSource which allows to bound how much data is being read from a source. It also contains a drainAll() function to discard the rest of the source, useful to keep the nix protocol in sync. --- src/libutil/serialise.hh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 969e4dff3..a344a5ac7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink From b226b5cd976ca71abb3c0861b56d5e5940430924 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 12 Aug 2019 17:19:43 +0200 Subject: [PATCH 133/433] nix-store: fix out of sync protocol If a NAR is already in the store, addToStore doesn't read the source which makes the protocol go out of sync. This happens for example when two client try to nix-copy-closure the same derivation at the same time. --- src/nix-store/nix-store.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f324056bb..0cbceb02f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success From 1dbaf119486fecbdf57ecc2f877307030b665f69 Mon Sep 17 00:00:00 2001 From: Toon Nolten Date: Sat, 17 Aug 2019 16:33:35 +0200 Subject: [PATCH 134/433] Fix nix-env documentation for --delete-generations The documentation for `--delete-generations` had an erroneous fullstop and as it turns out inaccurate information on the `+No.` syntax. --- doc/manual/command-ref/nix-env.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 693f23f7f..9d13d3014 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1370,10 +1370,10 @@ profile. The generations can be a list of generation numbers, the special value old to delete all non-current generations, a value such as 30d to delete all generations older than the specified number of days (except for the -generation that was active at that point in time), or a value such as. -+5 to only keep the specified items older than the -current generation. Periodically deleting old generations is important -to make garbage collection effective. +generation that was active at that point in time), or a value such as ++5 to only keep the specified number of generations. +Periodically deleting old generations is important to make garbage collection +effective. From 0463d5e36ffdbb60f851bd3b34756eff84151b28 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 22 Aug 2019 23:38:52 -0400 Subject: [PATCH 135/433] Allow empty /nix directory in multi-user installer With macOS catalina, we can no longer modify the root system volume (#2925). macOS provides a system configuration file in synthetic.conf(5) to create empty root directories. This can be used to mount /nix to a separate volume. As a result, this directory will need to already exist prior to installation. Instead, check for /nix/store and /nix/var for a live Nix installation. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9b757e7da..3d8b035eb 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -330,7 +330,7 @@ EOF fi done - if [ -d /nix ]; then + if [ -d /nix/store ] || [ -d /nix/var ]; then failure < Date: Fri, 23 Aug 2019 15:50:54 -0400 Subject: [PATCH 136/433] operators: document exact precedenc, split up similar operators --- doc/manual/expressions/language-operators.xml | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index f1f750934..1b14bc658 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -15,13 +15,16 @@ weakest binding). + Name Syntax Associativity Description + Precedence + Select e . attrpath [ or def ] @@ -33,19 +36,25 @@ weakest binding). dot-separated list of attribute names.) If the attribute doesn’t exist, return def if provided, otherwise abort evaluation. + 1 + Application e1 e2 left Call function e1 with argument e2. + 2 + Arithmetic Negation - e none Arithmetic negation. + 3 + Has Attribute e ? attrpath none @@ -53,34 +62,69 @@ weakest binding). the attribute denoted by attrpath; return true or false. + 4 + List Concatenation e1 ++ e2 right List concatenation. + 5 + Multiplication e1 * e2, + + left + Arithmetic multiplication. + 6 + + + Division + e1 / e2 left - Arithmetic multiplication and division. + Arithmetic division. + 6 + Addition + + e1 + e2 + + left + Arithmetic addition. + 7 + + + Subtraction - e1 + e2, e1 - e2 left - Arithmetic addition and subtraction. String or path concatenation (only by +). + Arithmetic subtraction. + 7 + String Concatenation + + string1 + string2 + + left + String concatenation. + 7 + + + Not ! e none Boolean negation. + 8 + Update e1 // e2 right @@ -89,47 +133,90 @@ weakest binding). e2 (with the latter taking precedence over the former in case of equally named attributes). + 9 + Less Than e1 < e2, - e1 > e2, - e1 <= e2, + + none + Arithmetic comparison. + 10 + + + Less Than or Equal To + + e1 <= e2 + + none + Arithmetic comparison. + 10 + + + Greater Than + + e1 > e2 + + none + Arithmetic comparison. + 10 + + + Greater Than or Equal To + e1 >= e2 none Arithmetic comparison. + 10 + Equality + + e1 == e2 + + none + Equality. + 11 + + + Inequality - e1 == e2, e1 != e2 none - Equality and inequality. + Inequality. + 11 + Logical And e1 && e2 left Logical AND. + 12 + Logical Or e1 || e2 left Logical OR. + 13 + Logical Implication e1 -> e2 none Logical implication (equivalent to !e1 || e2). + 14 -
\ No newline at end of file + From 5c06a8d3283139140e765b5f10ad7102a6a3e964 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 23 Aug 2019 20:24:39 -0400 Subject: [PATCH 137/433] Reset tmpDirInSandbox for unsandboxed --- src/libstore/build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0f71e7511..96e9b8edd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2330,6 +2330,7 @@ void DerivationGoal::startBuilder() int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { useChroot = false; + tmpDirInSandbox = tmpDir; goto fallback; } else if (res != 0) throw Error("unable to start build process"); From 6dab42a551a769f304c2a0e66e8771d7289502f2 Mon Sep 17 00:00:00 2001 From: Venkateswara Rao Mandela Date: Sat, 15 Jun 2019 00:27:26 +0530 Subject: [PATCH 138/433] installer: handle network proxy in systemd install If a network proxy configuration is detected, setup an override systemd unit file for nix-daemon service with the non-empty proxy variables. Proxy detection is performed by looking for http/https/ftp proxy and no proxy variables in user environment --- scripts/install-systemd-multi-user.sh | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) mode change 100644 => 100755 scripts/install-systemd-multi-user.sh diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh old mode 100644 new mode 100755 index 04bc539a1..bef3ac4f9 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat < Date: Fri, 23 Aug 2019 16:02:49 -0400 Subject: [PATCH 139/433] docs: document the installer's use of proxy env vars --- doc/manual/installation/env-variables.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index d1ee0bb2e..e2b8fc867 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon +
+ +Proxy Environment Variables + +The Nix installer has special handling for these proxy-related +environment variables: +http_proxy, https_proxy, +ftp_proxy, no_proxy, +HTTP_PROXY, HTTPS_PROXY, +FTP_PROXY, NO_PROXY. + +If any of these variables are set when running the Nix installer, +then the installer will create an override file at +/etc/systemd/system/nix-daemon.service.d/override.conf +so nix-daemon will use them. + +
+ From d45922472434783bdc29610479a735dd236229a3 Mon Sep 17 00:00:00 2001 From: Piotr Szubiakowski Date: Tue, 27 Aug 2019 10:35:35 +0200 Subject: [PATCH 140/433] nix-daemon.service: add install section. Signed-off-by: Piotr Szubiakowski --- misc/systemd/nix-daemon.service.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 5fc04a3f5..25655204d 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process + +[Install] +WantedBy=multi-user.target From 171d784404fefa7c91e2082f92ec92c39cc1aa8e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 27 Aug 2019 06:55:22 -0400 Subject: [PATCH 141/433] docs: operators: Make OR and AND capitalized --- doc/manual/expressions/language-operators.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index 1b14bc658..4f11bf529 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -190,7 +190,7 @@ weakest binding). 11 - Logical And + Logical AND e1 && e2 left @@ -198,7 +198,7 @@ weakest binding). 12 - Logical Or + Logical OR e1 || e2 left From 800fba1037ad97ff5e6d9acf9e820a30f0922f59 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 27 Aug 2019 10:57:09 -0400 Subject: [PATCH 142/433] Use wait4path on org.nixos.nix-daemon.plist When using a volume, the nix-daemon path may not exist. To avoid this issue, we must use the wait4path tool. This should solve one of the issues in multi-user on macOS Catalina. --- misc/launchd/org.nixos.nix-daemon.plist.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 92ed12fa5..6bc448d28 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -14,7 +14,12 @@ RunAtLoad Program - @bindir@/nix-daemon + /bin/sh + ProgramArguments + + -c + /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon + StandardErrorPath /var/log/nix-daemon.log StandardOutPath From 45b3dc325a1d49f86182195cf22fc96ddc05c644 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 17:00:04 +0200 Subject: [PATCH 143/433] Add 2.3 release notes --- doc/manual/release-notes/rl-2.3.xml | 70 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 428213b36..b0f7c8ec5 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,9 +4,23 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -Release 2.3 (????-??-??) +Release 2.3 (2019-08-??) -This release contains the following changes: +This is primarily a bug fix release. However, it makes some +incompatible changes: + + + + + Nix now uses BSD file locks instead of POSIX file + locks. Since previous releases used POSIX file locks, you should + not use Nix 2.2 and previous releases at the same time on a Nix + store. + + + + +It also has the following changes: @@ -18,5 +32,57 @@ already begin with refs/. + + + The installer now enables sandboxing by default on + Linux. The max-jobs setting now defaults to + 1. + + + + New builtin functions: + builtins.isPath, + builtins.hashFile. + + + + + nix: Add + () flag to + print build log output to stderr rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + + + + + Builds are now executed in a pseudo-terminal, and the + TERM evnironment variable is set to + xterm-256color. This allows many programs + (e.g. gcc, clang, + cmake) to print colorized log output. + + + + Add convenience flag. This flag + disables substituters; sets the tarball-ttl + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces. + + + + Add a post-build-hook setting to run a + program after a build has succeeded. + + + + Add a trace-function-calls setting to log + the duration of Nix function calls to stderr. + + + From 73728874ab89d77bce3b103acb239d4d5eec14b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 17:01:54 +0200 Subject: [PATCH 144/433] Hopefully fix post-hook test on macOS https://hydra.nixos.org/build/99262744 --- tests/push-to-store.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh index d97eb095d..6aadb916b 100755 --- a/tests/push-to-store.sh +++ b/tests/push-to-store.sh @@ -1,4 +1,4 @@ #!/bin/sh echo Pushing "$@" to "$REMOTE_STORE" -echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs From fdff96501f7f1762446a930e3a27b56a4fc912c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 21:18:00 +0200 Subject: [PATCH 145/433] Update release notes --- doc/manual/release-notes/rl-2.3.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index b0f7c8ec5..a1f4a4b75 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -83,6 +83,11 @@ incompatible changes: the duration of Nix function calls to stderr. + + On Linux, sandboxing is now disabled by default on systems + that don’t have the necessary kernel support. + + From ceefddafe839a0c9ac553367e16f965dda7fb086 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 22:18:34 +0200 Subject: [PATCH 146/433] Compress binary tarballs using xz Fixes https://github.com/NixOS/nix/issues/240. Apparently 'tar -xf' can decompress xz files on macOS nowadays. --- release.nix | 6 +++--- scripts/install.in | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/release.nix b/release.nix index 78b39108f..bec955730 100644 --- a/release.nix +++ b/release.nix @@ -165,10 +165,10 @@ let chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} - fn=$out/$dir.tar.bz2 + fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfj $fn \ + tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --absolute-names \ --hard-dereference \ @@ -295,7 +295,7 @@ let substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ") + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${build.x86_64-linux.src.version} diff --git a/scripts/install.in b/scripts/install.in index 4857638c0..902758b13 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -30,12 +30,11 @@ case "$(uname -s).$(uname -m)" in *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2" +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")" +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" -require_util bzcat "decompress the binary tarball" require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." @@ -57,7 +56,7 @@ fi unpack=$tmpDir/unpack mkdir -p "$unpack" -< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) From 5fa8b3f96577db65e080aaef65ad1cae462356cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 22:38:48 +0200 Subject: [PATCH 147/433] Update the libboost hack This cuts about 46 MiB from the closure. --- release.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index bec955730..ab677f3bd 100644 --- a/release.nix +++ b/release.nix @@ -72,7 +72,12 @@ let # https://github.com/NixOS/nixpkgs/issues/45462 '' mkdir -p $out/lib - cp ${boost}/lib/libboost_context* $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + chmod u+w $out/lib/*.so.* + ${lib.optionalString stdenv.isLinux '' + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} ''; configureFlags = configureFlags ++ From 7298a38a07d62d2077ca30a3ff619e32edb4eb1e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 16:29:44 +0200 Subject: [PATCH 148/433] Don't send certain setting overrides to the daemon These are already handled separately. This fixes warnings like warning: ignoring the user-specified setting 'max-jobs', because it is a restricted setting and you are not a trusted user when using the -j flag. --- src/libstore/remote-store.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 15faf78a5..1c2e23f9c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -191,6 +191,13 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map overrides; globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 693e68e09c9a17ca72b074bd2e575bf435647b45 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 28 Aug 2019 12:26:08 -0400 Subject: [PATCH 149/433] Set maximum name length in Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we allowed any length of name for Nix derivations. This is bad because different file systems have different max lengths. To make things predictable, I have picked a max. This was done by trying to build this derivation: derivation { name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; builder = "/no-such-path"; system = "x86_64-linux"; } Take off one a and it will not lead to file name too long. That ends up being 212 a’s. An even smaller max could be picked if we want to support more file systems. Working backwards, this is why: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-${name}.drv.chroot > 255 - 32 - 1 - 4 - 7 = 211 --- src/libstore/store-api.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f5608d384..3bb9db0b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -97,6 +97,10 @@ void checkStoreName(const string & name) reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || From c128031492b7134f67065b66a89f9183e79e99f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 22:04:16 +0200 Subject: [PATCH 150/433] Fix macOS build https://hydra.nixos.org/build/99500938 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index ab677f3bd..0fe683518 100644 --- a/release.nix +++ b/release.nix @@ -74,8 +74,8 @@ let mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - chmod u+w $out/lib/*.so.* ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} ''; From b6120d26a84a69755c049ca5115b4388239d4540 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 22:19:31 +0200 Subject: [PATCH 151/433] gc-auto.sh: Increase verbosity --- tests/gc-auto.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 1e91282d0..4b1f6374a 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -29,7 +29,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-A -L "($expr)" \ +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & pid=$! @@ -50,7 +50,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-B -L "($expr2)" \ +nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 wait "$pid" From f27e53f77e39d1247c47b48acb63af3d24ca74bf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:09:58 +0200 Subject: [PATCH 152/433] Cleanup --- src/libstore/gc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 366dbfb0a..f73601679 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -920,11 +920,11 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); - GCOptions options; options.maxFreed = settings.maxFree - avail; + printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCResults results; collectGarbage(options, results); From ecb0a23d51e46e2e0180850bdc8ad6cb7cc3d13f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:10:01 +0200 Subject: [PATCH 153/433] Add some more instrumentation --- tests/gc-auto.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 4b1f6374a..49894bc68 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -6,6 +6,9 @@ garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) +ls -l $garbage3 +du $garbage3 + fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free echo 1100 > $fake_free From 31f5ecfaa5bce2282f50daed1e4f477c64705fb3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:35:15 +0200 Subject: [PATCH 154/433] Maybe fix #3058 --- tests/gc-auto.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 49894bc68..68b7219f2 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -2,12 +2,12 @@ source common.sh clearStore -garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) -garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) -garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) ls -l $garbage3 -du $garbage3 +POSIXLY_CORRECT=1 du $garbage3 fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free From 5bdac86be2eb6d541784c4f149bfa06b59ef63a2 Mon Sep 17 00:00:00 2001 From: toonn Date: Thu, 29 Aug 2019 13:46:38 +0200 Subject: [PATCH 155/433] Reword to clarify newer generations are left alone My attempt at clarifying the docs resulted in a false explanation. This is now fixed and I added an example to eliminate all possible confusion. --- doc/manual/command-ref/nix-env.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 9d13d3014..c8c01f9a4 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1371,7 +1371,10 @@ special value old to delete all non-current generations, a value such as 30d to delete all generations older than the specified number of days (except for the generation that was active at that point in time), or a value such as -+5 to only keep the specified number of generations. ++5 to keep the last 5 generations +ignoring any newer than current, e.g., if 30 is the current +generation +5 will delete generation 25 +and all older generations. Periodically deleting old generations is important to make garbage collection effective. From a2c4fcd5e9782dc8d2998773380c7171ee53b813 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 14:49:58 +0200 Subject: [PATCH 156/433] Don't rely on st_blocks It doesn't seem very reliable on ZFS. --- src/libstore/gc.cc | 7 +++---- src/libutil/util.cc | 2 +- tests/gc-auto.sh | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f73601679..a166f4ee2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -690,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting '%1%'") % path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } @@ -701,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_blocks * 512ULL; + state.results.bytesFreed += st.st_size; } struct stat st; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 44fa72482..1b7449991 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -397,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 68b7219f2..54ead227e 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -22,7 +22,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... @@ -44,7 +44,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... From 87c604c1f04ff31a68f2fa73fe5a2ed9f6ec7ac7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 2 Sep 2019 18:35:10 -0400 Subject: [PATCH 157/433] Fix launchd program args launchd has some weird syntx. Apparently the program needs to be in the ProgramArguments, as Program appears to be ignored. --- misc/launchd/org.nixos.nix-daemon.plist.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 6bc448d28..b340610e9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -13,10 +13,9 @@ RunAtLoad - Program - /bin/sh ProgramArguments + /bin/sh -c /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon From e4ea3e03066a760c8cd462108af99aebaaa44c1b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 3 Sep 2019 07:32:44 +0200 Subject: [PATCH 158/433] docs: Note that tryEval doesn't do deep evaluation --- doc/manual/expressions/builtins.xml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4c1d618e9..f73d2e1ae 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -1607,12 +1607,18 @@ stdenv.mkDerivation (rec { builtins.tryEval e - Try to evaluate e. + Try to shallowly evaluate e. Return a set containing the attributes success (true if e evaluated successfully, false if an error was thrown) and value, equalling e - if successful and false otherwise. + if successful and false otherwise. Note that this + doesn't evaluate e deeply, so + let e = { x = throw ""; }; in (builtins.tryEval e).success + will be true. Using builtins.deepSeq + one can get the expected result: let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success will be + false. From 8c4ea7a4516c517a0dd37b446bf5c1a6b157064c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 12:51:14 +0200 Subject: [PATCH 159/433] Downloader: Remove a possible double call to Callback --- src/libstore/download.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index c322d267d..a7d059465 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -342,15 +342,9 @@ struct CurlDownloader : public Downloader (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); done = true; - - try { - act.progress(result.bodySize, result.bodySize); - callback(std::move(result)); - } catch (...) { - done = true; - callback.rethrow(); - } + callback(std::move(result)); } else { From 7348653ff4fc4e9b2dc24943aabdb57179b1c75a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 12:51:35 +0200 Subject: [PATCH 160/433] Ensure that Callback is called only once Also, make Callback movable but uncopyable. --- src/libstore/binary-cache-store.cc | 8 +++++--- src/libstore/download.cc | 6 +++--- src/libstore/http-binary-cache-store.cc | 12 +++++++----- src/libstore/store-api.cc | 8 +++++--- src/libutil/util.hh | 19 ++++++++++++++++--- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..e56be625d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -249,21 +249,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared(std::move(callback)); + getFile(narInfoFile, {[=](std::future> fut) { try { auto data = fut.get(); - if (!data) return callback(nullptr); + if (!data) return (*callbackPtr)(nullptr); stats.narInfoRead++; - callback((std::shared_ptr) + (*callbackPtr)((std::shared_ptr) std::make_shared(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index a7d059465..cdf56e09d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -77,13 +77,13 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, - Callback callback) + Callback && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) - , callback(callback) + , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { writtenToSink += len; @@ -665,7 +665,7 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared(*this, request, callback)); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index df2fb9332..e631d95f0 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -137,17 +137,19 @@ protected: auto request(makeRequest(path)); + auto callbackPtr = std::make_shared(std::move(callback)); + getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { + {[callbackPtr, this](std::future result) { try { - callback(result.get().data); + (*callbackPtr)(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); + return (*callbackPtr)(std::shared_ptr()); maybeDisable(); - callback.rethrow(); + callbackPtr->rethrow(); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3bb9db0b7..88a5b2f44 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -365,8 +365,10 @@ void Store::queryPathInfo(const Path & storePath, } catch (...) { return callback.rethrow(); } + auto callbackPtr = std::make_shared(std::move(callback)); + queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callback](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -386,8 +388,8 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - callback(ref(info)); - } catch (...) { callback.rethrow(); } + (*callbackPtr)(ref(info)); + } catch (...) { callbackPtr->rethrow(); } }}); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b538a0b41..686e81d3f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -445,21 +445,34 @@ string get(const T & map, const string & key, const string & def = "") type T or an exception. (We abuse std::future to pass the value or exception.) */ template -struct Callback +class Callback { std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: Callback(std::function)> fun) : fun(fun) { } - void operator()(T && t) const + Callback(Callback && callback) : fun(std::move(callback.fun)) { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) + { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_value(std::move(t)); fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) const + void rethrow(const std::exception_ptr & exc = std::current_exception()) { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_exception(exc); fun(promise.get_future()); From f186000367978fbe590343f47951232f42a30bec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 13:00:55 +0200 Subject: [PATCH 161/433] Add some noexcepts This is to assert that callback functions should never throw (since the context in which they're called may not be able to handle the exception). --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/binary-cache-store.hh | 4 ++-- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 9 +++++---- src/libstore/store-api.hh | 4 ++-- src/libutil/util.hh | 4 ++-- 11 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e56be625d..10cde8704 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -55,7 +55,7 @@ void BinaryCacheStore::init() } void BinaryCacheStore::getFile(const std::string & path, - Callback> callback) + Callback> callback) noexcept { try { callback(getFile(path)); @@ -240,7 +240,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { auto uri = getUri(); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 953f3b90a..af108880c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -47,7 +47,7 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - Callback> callback); + Callback> callback) noexcept; std::shared_ptr getFile(const std::string & path); @@ -73,7 +73,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e631d95f0..779f89e68 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -131,7 +131,7 @@ protected: } void getFile(const std::string & path, - Callback> callback) override + Callback> callback) noexcept override { checkEnabled(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7c9bc2b68..d5fbdd25a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback> callback) override + Callback> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63b11467e..2fcf08491 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -629,7 +629,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { auto info = std::make_shared(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index af8b84bf5..3ae34c403 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c2e23f9c..e38fe49a7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -349,7 +349,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80f18ab71..82fbec092 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -41,7 +41,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 88a5b2f44..5f63c53b5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -329,13 +329,14 @@ ref Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); { auto res = state.lock()->pathInfoCache.get(hashPart); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 599677376..7fb568602 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,12 +360,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback> callback); + Callback> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback> callback) = 0; + Callback> callback) noexcept = 0; public: diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 686e81d3f..07c3d28ff 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -460,7 +460,7 @@ public: if (prev) done.test_and_set(); } - void operator()(T && t) + void operator()(T && t) noexcept { auto prev = done.test_and_set(); assert(!prev); @@ -469,7 +469,7 @@ public: fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept { auto prev = done.test_and_set(); assert(!prev); From cec50290bf5537d574fd94557bd3918f2bea8a30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 15:45:32 +0200 Subject: [PATCH 162/433] gc-auto.sh: Add some more instrumentation --- tests/gc-auto.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 54ead227e..5c8be7c0e 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -17,6 +17,7 @@ expr=$(cat < \$out/bar @@ -32,14 +33,11 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ - --min-free 1000 --max-free 2000 --min-free-check-interval 1 & -pid=$! - expr2=$(cat < \$out/bar echo 1... @@ -53,6 +51,10 @@ with import ./config.nix; mkDerivation { EOF ) +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 From e07ec8d27e08bf23eccab079b044a6f1b37f3ac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 16:02:12 +0200 Subject: [PATCH 163/433] Support allowSubstitutes attribute in structured attribute derivations Hopefully fixes #3081 (didn't test). --- src/libstore/build.cc | 2 +- src/libstore/derivations.cc | 6 ------ src/libstore/derivations.hh | 2 -- src/libstore/misc.cc | 4 +++- src/libstore/parsed-derivations.cc | 5 +++++ src/libstore/parsed-derivations.hh | 2 ++ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index be52b66a7..ab725e8e9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1197,7 +1197,7 @@ void DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3961126ff..23fcfb281 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::substitutesAllowed() const -{ - return get(env, "allowSubstitutes", "1") == "1"; -} - - bool BasicDerivation::isBuiltin() const { return string(builder, 0, 8) == "builtin:"; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9753e796d..8e02c9bc5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,8 +56,6 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool substitutesAllowed() const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index adcce026f..dddf13430 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets, } Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); PathSet invalid; for (auto & j : drv.outputs) @@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets, invalid.insert(j.second.path); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, i2.first, make_ref(drv), output, drvState)); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 17fde00a0..87be8a24e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const return getBoolAttr("preferLocalBuild") && canBuildLocally(); } +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index ed07dc652..9bde4b4dc 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -30,6 +30,8 @@ public: bool canBuildLocally() const; bool willBuildLocally() const; + + bool substitutesAllowed() const; }; } From 08ee36495087e3d6f573615c050497ec2298cafa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 18:11:43 +0200 Subject: [PATCH 164/433] gc-auto.sh: More test fixes --- tests/gc-auto.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 5c8be7c0e..87f1d6798 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -23,10 +23,13 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 200 > $fake_free + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free echo 2... - sleep 2 + sleep 1 echo 3... + sleep 1 + echo 4... [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] ''; } @@ -42,10 +45,13 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 200 > $fake_free + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free echo 2... - sleep 2 + sleep 1 echo 3... + sleep 1 + echo 4... ''; } EOF From 5fad9d01c242ffa9d83cddeeacb9abbaa3848c7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 12:52:54 +0200 Subject: [PATCH 165/433] gc-auto.sh: Increase sleep time --- tests/gc-auto.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 87f1d6798..de1e2cfe4 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -26,9 +26,9 @@ with import ./config.nix; mkDerivation { echo 200 > ${fake_free}.tmp1 mv ${fake_free}.tmp1 $fake_free echo 2... - sleep 1 + sleep 2 echo 3... - sleep 1 + sleep 2 echo 4... [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] ''; @@ -48,9 +48,9 @@ with import ./config.nix; mkDerivation { echo 200 > ${fake_free}.tmp2 mv ${fake_free}.tmp2 $fake_free echo 2... - sleep 1 + sleep 2 echo 3... - sleep 1 + sleep 2 echo 4... ''; } From b774845af7a645b44bff69cf9f655c47fe4b9fb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 12:53:22 +0200 Subject: [PATCH 166/433] Set release date --- doc/manual/release-notes/rl-2.3.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index a1f4a4b75..291b56448 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -Release 2.3 (2019-08-??) +Release 2.3 (2019-09-04) This is primarily a bug fix release. However, it makes some incompatible changes: From 252c78b288426bae2f379e8320d396b7eb40253b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:07 +0200 Subject: [PATCH 167/433] Tweak release notes --- doc/manual/release-notes/rl-2.3.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 291b56448..80fbd6396 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -13,9 +13,8 @@ incompatible changes: Nix now uses BSD file locks instead of POSIX file - locks. Since previous releases used POSIX file locks, you should - not use Nix 2.2 and previous releases at the same time on a Nix - store. + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store. @@ -47,9 +46,9 @@ incompatible changes: - nix: Add + The nix command has a new () flag to - print build log output to stderr rather than showing the last log + print build log output to stderr, rather than showing the last log line in the progress bar. To distinguish between concurrent builds, log lines are prefixed by the name of the package. @@ -57,7 +56,7 @@ incompatible changes: Builds are now executed in a pseudo-terminal, and the - TERM evnironment variable is set to + TERM environment variable is set to xterm-256color. This allows many programs (e.g. gcc, clang, cmake) to print colorized log output. From d20f814cdef55fd9d7cecf464b32fb8b5efecb1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:33 +0200 Subject: [PATCH 168/433] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c0943d3e9..7208c2182 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3 \ No newline at end of file +2.4 \ No newline at end of file From 5dafde28dbec379678da6d033cdc5c48856babf5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 19:24:35 +0200 Subject: [PATCH 169/433] BinaryCacheStore: Add index-debug-info option This integrates the functionality of the index-debuginfo program in nixos-channel-scripts to maintain an index of DWARF debuginfo files in a format usable by dwarffs. Thus the debug info index is updated by Hydra rather than by the channel mirroring script. Example usage: $ nix copy --to 'file:///tmp/binary-cache?index-debug-info=true' /nix/store/vr9mhcch3fljzzkjld3kvkggvpq38cva-nix-2.2.2-debug $ cat /tmp/binary-cache/debuginfo/036b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug {"archive":"../nar/0313h2kdhk4v73xna9ysiksp2v8xrsk5xsw79mmwr3rg7byb4ka8.nar.xz","member":"lib/debug/.build-id/03/6b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug"} Fixes #3083. --- src/libstore/binary-cache-store.cc | 80 ++++++++++++++++++++---- src/libstore/binary-cache-store.hh | 1 + src/libstore/local-binary-cache-store.cc | 2 + 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 10cde8704..8e6f1f55d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,10 +10,13 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include - #include +#include + +#include namespace nix { @@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, narAccessor); - { auto res = jsonRoot.placeholder("root"); listNar(res, narAccessor, "", true); @@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, makeNarAccessor(nar)); - } - /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refsize() / nar->size()) * 100.0) % duration); - /* Atomically write the NAR file. */ narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index af108880c..c77292294 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,6 +17,7 @@ public: const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting parallelCompression{this, false, "parallel-compression", diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b7001795b..9f99ee76d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -63,6 +63,8 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } From b9d6c6f000d2ba3efd7c124e46982425962f4fbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 21:44:40 +0200 Subject: [PATCH 170/433] upload-release.pl: Fix sshfs call --- maintainers/upload-release.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 1cdf5ed16..77534babb 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n"; File::Path::make_path($releasesDir); if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; + system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; } my $releaseDir = "$releasesDir/nix/$releaseName"; From a56b51a0ba7b0d6fdff7fd0127a118185b146f4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 14:14:03 +0200 Subject: [PATCH 171/433] Disable OpenSSL lock callback on OpenSSL >= 1.1.1 --- src/libmain/shared.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0afddfb78..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); From 92ede15dd902f7c1d2771c194b8bb73fe406840f Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:11:37 +0200 Subject: [PATCH 172/433] docs: Fix a typo in github in an example --- doc/manual/expressions/builtins.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4c1d618e9..39407c9ac 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -448,7 +448,7 @@ stdenv.mkDerivation { … } Fetching an arbitrary ref builtins.fetchGit { - url = "https://gitub.com/NixOS/nix.git"; + url = "https://github.com/NixOS/nix.git"; ref = "refs/heads/0.5-release"; } From ae244af242ca3621e5a3b9196f27d9fcbf297266 Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:18:47 +0200 Subject: [PATCH 173/433] docs: Use the explicit ref for fetchGit with a tag With the merge of #2582, the syntax "tags/1.9" for refs does not work anymore. However, the new syntax "refs/tags/1.9" seems to support annotated tags, such as "refs/tags/2.0". Closes #2385. --- doc/manual/expressions/builtins.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 39407c9ac..191fa7810 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -499,11 +499,8 @@ stdenv.mkDerivation { … } Fetching a tag builtins.fetchGit { url = "https://github.com/nixos/nix.git"; - ref = "tags/1.9"; + ref = "refs/tags/1.9"; } - Due to a bug (#2385), - only non-annotated tags can be fetched. From 6b83174ffffbdfc3f876d94d5178e0b83f675cae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Sep 2019 20:05:27 +0200 Subject: [PATCH 174/433] std::uncaught_exception() -> std::uncaught_exceptions() The former is deprecated in C++17. Fixes a clang warning. --- src/libstore/remote-store.cc | 2 +- src/libutil/json.hh | 2 +- src/libutil/util.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e38fe49a7..89b699414 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -228,7 +228,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exception()) { + if (!daemonException && std::uncaught_exceptions()) { handle.markBad(); debug("closing daemon connection because of an exception"); } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..45a22f011 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -170,7 +170,7 @@ public: ~JSONPlaceholder() { - assert(!first || std::uncaught_exception()); + assert(!first || std::uncaught_exceptions()); } template diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1b7449991..e65f8ee56 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1169,7 +1169,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exception()) { + if (!interruptThrown && !std::uncaught_exceptions()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } From 9533d85ce009c65722a2addc55b64ee51f15dea3 Mon Sep 17 00:00:00 2001 From: Alexandre Esteves Date: Tue, 17 Sep 2019 23:40:49 +0100 Subject: [PATCH 175/433] Move 'builtins.splitVersion' to position respecting alphabetical order --- doc/manual/expressions/builtins.xml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 191fa7810..9bd86e120 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -170,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - - builtins.splitVersion - s - - Split a string representing a version into its - components, by the same version splitting logic underlying the - version comparison in - nix-env -u. - - - - builtins.concatLists lists @@ -1272,6 +1260,19 @@ Evaluates to [ " " [ "FOO" ] " " ]. + + + builtins.splitVersion + s + + Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in + nix-env -u. + + + + builtins.stringLength e From 619cc4af855fab7b0400586a4fd40745b23e72ad Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 18 Sep 2019 11:40:11 +0000 Subject: [PATCH 176/433] function-trace: always show the trace If the user invokes nix with --trace-function-calls it means that they want to see the trace. --- contrib/stack-collapse.py | 5 ++--- src/libexpr/function-trace.hh | 4 ++-- tests/function-trace.sh | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py index ee77e1589..f5602c95c 100755 --- a/contrib/stack-collapse.py +++ b/contrib/stack-collapse.py @@ -1,12 +1,11 @@ #!/usr/bin/env nix-shell #!nix-shell -i python3 -p python3 --pure -# To be used with `--trace-function-calls` and `-vvvv` and -# `flamegraph.pl`. +# To be used with `--trace-function-calls` and `flamegraph.pl`. # # For example: # -# nix-instantiate --trace-function-calls -vvvv '' -A hello 2> nix-function-calls.trace +# nix-instantiate --trace-function-calls '' -A hello 2> nix-function-calls.trace # ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded # nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index 8234b7603..8b0ec848d 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -12,13 +12,13 @@ struct FunctionCallTrace FunctionCallTrace(const Pos & pos) : pos(pos) { auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto ns = std::chrono::duration_cast(duration); - vomit("function-trace entered %1% at %2%", pos, ns.count()); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); } ~FunctionCallTrace() { auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto ns = std::chrono::duration_cast(duration); - vomit("function-trace exited %1% at %2%", pos, ns.count()); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); } }; } diff --git a/tests/function-trace.sh b/tests/function-trace.sh index f7d93b435..182a4d5c2 100755 --- a/tests/function-trace.sh +++ b/tests/function-trace.sh @@ -8,7 +8,6 @@ expect_trace() { actual=$( nix-instantiate \ --trace-function-calls \ - -vvvv \ --expr "$expr" 2>&1 \ | grep "function-trace" \ | sed -e 's/ [0-9]*$//' From c6a542f22a165f711fc0730bfcfbd936ad1c3dfb Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 21 Sep 2019 18:53:15 -0400 Subject: [PATCH 177/433] Fix spelling in comment --- src/libstore/build.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ab725e8e9..9a6729f9e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2364,7 +2364,7 @@ void DerivationGoal::startBuilder() child = clone(childEntry, stack + stackSize, flags, this); } if (child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unpriveleged + /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. */ @@ -2410,8 +2410,7 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); - /* Signal the builder that we've updated its user - namespace. */ + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; From e63c9e73e3e5d1f31fa5065c9ff59f442dd07d0e Mon Sep 17 00:00:00 2001 From: zimbatm Date: Sun, 22 Sep 2019 10:57:20 +0000 Subject: [PATCH 178/433] libstore: don't forward --show-trace --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c2e23f9c..76107db54 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -198,6 +198,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.maxSilentTime.name); overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From bd79c1f6f6391786772a8a79962abe22f374cca4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Sep 2019 21:29:33 +0200 Subject: [PATCH 179/433] Don't catch exceptions by value (cherry picked from commit 893be6f5e36abb58bbaa9c49055a5218114dd514) --- src/libexpr/json-to-value.cc | 4 ++-- src/libexpr/primops/fetchGit.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/nix-build/nix-build.cc | 2 +- src/nix/edit.cc | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 3f6017957..8bae986f9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkFloat(v, stod(tmp_number)); else mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range e) { + } catch (std::out_of_range & e) { throw JSONParseError("out-of-range JSON number"); } } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 6229fef8d..90f600284 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -38,7 +38,7 @@ GitInfo exportGit(ref store, const std::string & uri, try { runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError e) { + } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; clean = false; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2fcf08491..a2af51d0e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -879,8 +879,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { - } catch (SubstituterDisabled) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) printError(e.what()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c6a4d4166..33ad28704 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -280,7 +280,7 @@ static void _main(int argc, char * * argv) auto absolute = i; try { absolute = canonPath(absPath(i), true); - } catch (Error e) {}; + } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d..a6169f118 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -55,7 +55,7 @@ struct CmdEdit : InstallableCommand int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } From 3a022d45993b6fa8c7bf03517a3a3d1a2ab15f4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 13:48:53 +0200 Subject: [PATCH 180/433] Shut up some warnings (cherry picked from commit 99e8e58f2de9941353b47ed14fbe4ed76d635519) --- src/libstore/download.hh | 2 ++ src/libstore/fs-accessor.hh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c68381846..68565bf46 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -91,6 +91,8 @@ class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; From 10bfc5c0d09d5508e3dab4c32f3368caeb5f7f56 Mon Sep 17 00:00:00 2001 From: Joseph Lucas Date: Mon, 23 Sep 2019 13:18:59 +0000 Subject: [PATCH 181/433] Update garbage-collection.xml readability 1. remove a typo space 2. Simplify negative style by using affirmative style --- doc/manual/packages/garbage-collection.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml index a1b0ef22a..b506f22b0 100644 --- a/doc/manual/packages/garbage-collection.xml +++ b/doc/manual/packages/garbage-collection.xml @@ -52,12 +52,13 @@ garbage collector as follows: $ nix-store --gc -The behaviour of the gargage collector is affected by the keep- -derivations (default: true) and keep-outputs +The behaviour of the gargage collector is affected by the +keep-derivations (default: true) and keep-outputs (default: false) options in the Nix configuration file. The defaults will ensure -that all derivations that are not build-time dependencies of garbage collector roots -will be collected but that all output paths that are not runtime dependencies -will be collected. (This is usually what you want, but while you are developing +that all derivations that are build-time dependencies of garbage collector roots +will be kept and that all output paths that are runtime dependencies +will be kept as well. All other derivations or paths will be collected. +(This is usually what you want, but while you are developing it may make sense to keep outputs to ensure that rebuild times are quick.) If you are feeling uncertain, you can also first view what files would From 168a8879165dd0deab1a93d343a6003146f37031 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Jun 2019 10:34:13 +0200 Subject: [PATCH 182/433] Fix fetchTarball with chroot stores Fixes #2405. --- src/libexpr/primops.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 350dba474..d4c60f870 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2089,12 +2089,12 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, request).path; + auto res = getDownloader()->downloadCached(state.store, request); if (state.allowedPaths) - state.allowedPaths->insert(res); + state.allowedPaths->insert(res.path); - mkString(v, res, PathSet({res})); + mkString(v, res.storePath, PathSet({res.storePath})); } From 74b4737d8f0e1922ef5314a158271acf81cd79f8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:07:07 -0400 Subject: [PATCH 183/433] Add libatomic for 32-bit ARM Fixes #3113 --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index a52830b38..96a2065b1 100644 --- a/configure.ac +++ b/configure.ac @@ -157,6 +157,11 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" +# Boost atomic needs GCC libatomic on 32-bit ARM +case "$host_cpu" in + armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" +esac + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From b1c34152feac76c9baded58732068541b7a586f1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:22:18 -0400 Subject: [PATCH 184/433] Use more robust test for libatomics Taken from Mesa configure script: https://github.com/mesa3d/mesa/blob/17.2/configure.ac#L405-L427 --- configure.ac | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 96a2065b1..ebe6d4267 100644 --- a/configure.ac +++ b/configure.ac @@ -157,11 +157,18 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" -# Boost atomic needs GCC libatomic on 32-bit ARM -case "$host_cpu" in - armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" -esac - +# On some platforms, new-style atomics need a helper library +AC_MSG_CHECKING(whether -latomic is needed) +AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#include +uint64_t v; +int main() { + return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE); +}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) +AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) +if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then + LIBS="-latomic $LIBS" +fi # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From 15e70c662e86c42658cdbc76b54da0f3f2be3c30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Oct 2019 16:26:15 +0200 Subject: [PATCH 185/433] Fix indentation --- src/nix-build/nix-build.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 33ad28704..2b36846cd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -282,12 +282,12 @@ static void _main(int argc, char * * argv) absolute = canonPath(absPath(i), true); } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(*state, store, absolute)); - else - /* If we're in a #! script, interpret filenames - relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); } } From 93b1ce1ac522ff92626da5cdd689008063935844 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 16:34:59 +0200 Subject: [PATCH 186/433] Revert "std::uncaught_exception() -> std::uncaught_exceptions()" This reverts commit 6b83174ffffbdfc3f876d94d5178e0b83f675cae because it doesn't work on macOS yet. https://hydra.nixos.org/build/102617587 --- src/libstore/remote-store.cc | 2 +- src/libutil/json.hh | 2 +- src/libutil/util.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b9e7a80ba..b7f202a6b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -229,7 +229,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exceptions()) { + if (!daemonException && std::uncaught_exception()) { handle.markBad(); debug("closing daemon connection because of an exception"); } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 45a22f011..02a39917f 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -170,7 +170,7 @@ public: ~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + assert(!first || std::uncaught_exception()); } template diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e65f8ee56..1b7449991 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1169,7 +1169,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { + if (!interruptThrown && !std::uncaught_exception()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } From c5bd564c69ec1fb4f2ffc1091653d99ebc87e284 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sun, 6 Oct 2019 16:54:56 -0400 Subject: [PATCH 187/433] nix doctor: add more logging output to checks When running nix doctor on a healthy system, it just prints the store URI and nothing else. This makes it unclear whether the system is in a good state and what check(s) it actually ran, since some of the checks are optional depending on the store type. This commit updates nix doctor to print an colored log message for every check that it does, and explicitly state whether that check was a PASS or FAIL to make it clear to the user whether the system passed its checkup with the doctor. Fixes #3084 --- src/nix/doctor.cc | 67 +++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7b5444619..795236c5f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -1,11 +1,17 @@ +#include + #include "command.hh" +#include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" +#include "util.hh" #include "worker-protocol.hh" using namespace nix; +namespace { + std::string formatProtocol(unsigned int proto) { if (proto) { @@ -16,6 +22,18 @@ std::string formatProtocol(unsigned int proto) return "unknown"; } +bool checkPass(const std::string & msg) { + logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + return true; +} + +bool checkFail(const std::string & msg) { + logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + return false; +} + +} + struct CmdDoctor : StoreCommand { bool success = true; @@ -27,13 +45,12 @@ struct CmdDoctor : StoreCommand std::string description() override { - return "check your system for potential problems"; + return "check your system for potential problems and print a PASS or FAIL for each check."; } void run(ref store) override { - std::cout << "Store uri: " << store->getUri() << std::endl; - std::cout << std::endl; + logger->log("Running checks against store uri: " + store->getUri()); auto type = getStoreType(); @@ -56,15 +73,14 @@ struct CmdDoctor : StoreCommand dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); if (dirs.size() != 1) { - std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; - std::cout << std::endl; + std::stringstream ss; + ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; + ss << " " << dir << "\n"; + return checkFail(ss.str()); } - return true; + return checkPass("PATH contains only one nix version."); } bool checkProfileRoots(ref store) @@ -87,17 +103,17 @@ struct CmdDoctor : StoreCommand } if (!dirs.empty()) { - std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; - std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; - std::cout << "garbage collected, resulting in broken symlinks." << std::endl; - std::cout << std::endl; + std::stringstream ss; + ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n" + << "The generation this profile points to might not have a gcroot and could be\n" + << "garbage collected, resulting in broken symlinks.\n\n"; for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; + ss << " " << dir << "\n"; + ss << "\n"; + return checkFail(ss.str()); } - return true; + return checkPass("All profiles are gcroots."); } bool checkStoreProtocol(unsigned int storeProto) @@ -107,17 +123,16 @@ struct CmdDoctor : StoreCommand : PROTOCOL_VERSION; if (clientProto != storeProto) { - std::cout << "Warning: protocol version of this client does not match the store." << std::endl; - std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; - std::cout << "sync with the daemon." << std::endl; - std::cout << std::endl; - std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; - std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; - std::cout << std::endl; - return false; + std::stringstream ss; + ss << "Warning: protocol version of this client does not match the store.\n" + << "While this is not necessarily a problem it's recommended to keep the client in\n" + << "sync with the daemon.\n\n" + << "Client protocol: " << formatProtocol(clientProto) << "\n" + << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; + return checkFail(ss.str()); } - return true; + return checkPass("Client protocol matches store protocol."); } }; From 6f6cb5e3880d0c7a1dd2bc13c2e0be8ce0ae9fa1 Mon Sep 17 00:00:00 2001 From: Sam Doshi Date: Mon, 7 Oct 2019 11:40:42 +0100 Subject: [PATCH 188/433] nix search: remove verbose example --- src/nix/search.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index e086de226..764dcb70a 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -80,10 +80,6 @@ struct CmdSearch : SourceExprCommand, MixJSON Example{ "To search for git and frontend or gui:", "nix search git 'frontend|gui'" - }, - Example{ - "To display the description of the found packages:", - "nix search git --verbose" } }; } From 00a567588e43b51944a9b91c1e890bda27773aed Mon Sep 17 00:00:00 2001 From: Danny Bautista Date: Tue, 8 Oct 2019 14:02:40 -0400 Subject: [PATCH 189/433] Fix typos in the Nix Manual. --- doc/manual/command-ref/conf-file.xml | 2 +- doc/manual/command-ref/nix-env.xml | 2 +- doc/manual/command-ref/nix-prefetch-url.xml | 2 +- doc/manual/expressions/builtins.xml | 4 ++-- doc/manual/expressions/simple-building-testing.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 320e15339..48dce7c95 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -433,7 +433,7 @@ builtins.fetchurl { keep-env-derivations If false (default), derivations - are not stored in Nix user environments. That is, the derivation + are not stored in Nix user environments. That is, the derivations of any build-time-only dependencies may be garbage-collected. If true, when you add a Nix derivation to diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index c8c01f9a4..d257a5e49 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -659,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4' gcc-3.3.1 are split into two parts: the package name (gcc), and the version (3.3.1). The version part starts after the first -dash not following by a letter. x is considered an +dash not followed by a letter. x is considered an upgrade of y if their package names match, and the version of y is higher that that of x. diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml index 8ef748c74..621ded72e 100644 --- a/doc/manual/command-ref/nix-prefetch-url.xml +++ b/doc/manual/command-ref/nix-prefetch-url.xml @@ -53,7 +53,7 @@ avoided. If hash is specified, then a download is not performed if the Nix store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an -error if signaled if the actual hash of the file does not match the +error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. Additionally, diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index e164d1441..465fa1e0b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -289,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" Return element n from the list xs. Elements are counted - starting from 0. A fatal error occurs in the index is out of + starting from 0. A fatal error occurs if the index is out of bounds. @@ -1466,7 +1466,7 @@ in foo A set containing { __toString = self: ...; }. An integer. A list, in which case the string representations of its elements are joined with spaces. - A Boolean (false yields "", true yields "1". + A Boolean (false yields "", true yields "1"). null, which yields the empty string. diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index 0348c082b..7326a3e76 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -43,7 +43,7 @@ use nix-build’s switch to give the symlink another name. -Nix has a transactional semantics. Once a build finishes +Nix has transactional semantics. Once a build finishes successfully, Nix makes a note of this in its database: it registers that the path denoted by out is now valid. If you try to build the derivation again, Nix From 0847f2f1b3145a62dc34707ba788275ce6b6fc57 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 21:17:27 -0400 Subject: [PATCH 190/433] Copy instead of linking launch agent On Catalina, the /nix filesystem might not be mounted at start time. To avoid this service not starting, we need to keep the launch agent outside of the Nix store. A wait4pid will hold for our /nix dir to be mounted. Fixes #3125. --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- scripts/install-darwin-multi-user.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index b340610e9..39d7757a4 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -17,7 +17,7 @@ /bin/sh -c - /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon + /bin/wait4path @bindir@/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 87c4c2b05..49076bd5c 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -39,7 +39,7 @@ EOF poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon as a LaunchDaemon" \ - ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" + cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist From d4e51aac080e73982e16b43088f9ae404f07136e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 21:35:09 -0400 Subject: [PATCH 191/433] Make preexisting Nix install a warning, not a failure In the multi-user install script, we originally made sure no previous references to Nix existed. This prevented any previous installs from contaminating the new install. However, some users need the ability to repair their existing Nix installation without uninstalling all references to Nix. This change allows users with existing Nix installations to use the installer, while still outputing a warning message on the dangers of this. As a result, the multi-user install script work much more like the single-user install script has worked in the past. This is a requirement for macOS Catalina users now that /Library/LaunchDaemons/org.nixos.nix-daemon.plisg is not managed by the Nix store. If there is ever a change to the .plist, all users will need to rerun this install script to get the new changes. Otherwise, changes to the launch daemon will require manual interventions. --- scripts/install-multi-user.sh | 104 ++-------------------------------- 1 file changed, 5 insertions(+), 99 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a41309e93..fd0f54184 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -278,73 +278,9 @@ EOF fi if type nix-env 2> /dev/null >&2; then - failure < Date: Tue, 8 Oct 2019 22:41:05 -0400 Subject: [PATCH 192/433] =?UTF-8?q?Don=E2=80=99t=20source=20bashrc=20in=20?= =?UTF-8?q?pure=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure mode should not try to source the user’s bashrc file. These may have many impurities that the user does not expect to get into their shell. Fixes #3090 --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 2b36846cd..3e1081c05 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -412,7 +412,7 @@ static void _main(int argc, char * * argv) auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( (keepTmp ? "" : "rm -rf '%1%'; "s) + - "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " From 199e888785bd23073e44e56f6c74b95dc7c10ffa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 23:12:31 -0400 Subject: [PATCH 193/433] Handle empty sandbox_shell Previously, SANDBOX_SHELL was set to empty when unavailable. This caused issues when actually generating the sandbox. Instead, just set SANDBOX_SHELL when --with-sandbox-shell= is non-empty. Alternative implementation to https://github.com/NixOS/nix/pull/3038. --- src/libstore/local.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 89fc918c3..d690fea28 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -39,9 +39,12 @@ libstore_CXXFLAGS = \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ - -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" +ifneq ($(sandbox_shell),) +libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" +endif + $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/build.cc: From 8c4a5e7ba1990348b9497c0fc4dc236dda3e7986 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Wed, 9 Oct 2019 08:51:49 +0100 Subject: [PATCH 194/433] Make nix-daemon.plist less fragile on macOS We're calling `wait4path` on the full, resolved `@bindir@/nix-daemon` path. That means we're hardcoding something like: /bin/wait4path /nix/store/zs9c5xhp3zv9p23qnjxp87nl5injsi1i-nix-2.3/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon That seems unnecessarily fragile. It might be better to wait4path on the path we intend to call. --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 39d7757a4..9f26296a9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -17,7 +17,7 @@ /bin/sh -c - /bin/wait4path @bindir@/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log From 64d88729005841ff3ccfa75134f9aa5e5828da6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 14:46:44 +0200 Subject: [PATCH 195/433] nix-build: Fix compilation --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3e1081c05..085650e4f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -412,7 +412,7 @@ static void _main(int argc, char * * argv) auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( (keepTmp ? "" : "rm -rf '%1%'; "s) + - (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " From 7c74f075f4a7274ad38c90085cc269a19a977438 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 14:46:58 +0200 Subject: [PATCH 196/433] nix search: Don't quietly ignore errors --- src/nix/search.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/search.cc b/src/nix/search.cc index 764dcb70a..eb75493e4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -261,6 +261,7 @@ struct CmdSearch : SourceExprCommand, MixJSON https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ if (!jsonCacheFile) throw Error("error writing to %s", tmpFile); + throw; } if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) From e6e61f0a54dac0174df996e93fcfedcac7769ab4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 15:36:51 +0200 Subject: [PATCH 197/433] getSourceExpr(): Handle channels Fixes #1892. Fixes #1865. Fixes #3119. --- src/nix/installables.cc | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0c1ad3ab3..52f9778fc 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -45,25 +45,26 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) std::unordered_set seen; - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); -#if 0 - auto res = state.resolveSearchPathElem(i); - if (!res.first) continue; - if (!pathExists(res.second)) continue; - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), res.second)); -#endif + auto addEntry = [&](const std::string & name) { + if (name == "") return; + if (!seen.insert(name).second) return; Value * v1 = state.allocValue(); mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); - } + }; + + for (auto & i : searchPath) + /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto & j : readDirectory(i.second)) + if (j.name != "manifest.nix" + && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); vSourceExpr->attrs->sort(); } From 99b73fb5071db9fd757c9927fc3fde34e2abac63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 15:51:52 +0200 Subject: [PATCH 198/433] OCD performance fix: {find,count}+insert => insert --- src/libexpr/eval.cc | 24 +++++++------------ src/libexpr/get-drvs.cc | 3 +-- src/libexpr/parser.y | 3 +-- src/libexpr/primops.cc | 6 ++--- src/libexpr/value-to-xml.cc | 5 ++-- src/libstore/build.cc | 11 ++++----- src/libstore/builtins/buildenv.cc | 3 +-- src/libstore/local-store.cc | 3 +-- src/libstore/misc.cc | 9 +++---- src/libstore/references.cc | 3 +-- src/libstore/store-api.cc | 3 +-- src/nix-env/nix-env.cc | 7 ++---- src/nix-store/dotgraph.cc | 7 ++---- src/nix-store/nix-store.cc | 3 +-- src/nix/verify.cc | 3 +-- .../resolve-system-dependencies.cc | 4 +--- 16 files changed, 32 insertions(+), 65 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9f4b6b411..f60e8d3ab 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -46,11 +46,10 @@ static void printValue(std::ostream & str, std::set & active, con { checkInterrupt(); - if (active.find(&v) != active.end()) { + if (!active.insert(&v).second) { str << ""; return; } - active.insert(&v); switch (v.type) { case tInt: @@ -1446,8 +1445,7 @@ void EvalState::forceValueDeep(Value & v) std::function recurse; recurse = [&](Value & v) { - if (seen.find(&v) != seen.end()) return; - seen.insert(&v); + if (!seen.insert(&v).second) return; forceValue(v); @@ -1865,8 +1863,7 @@ size_t valueSize(Value & v) std::set seen; auto doString = [&](const char * s) -> size_t { - if (seen.find(s) != seen.end()) return 0; - seen.insert(s); + if (!seen.insert(s).second) return 0; return strlen(s) + 1; }; @@ -1874,8 +1871,7 @@ size_t valueSize(Value & v) std::function doEnv; doValue = [&](Value & v) -> size_t { - if (seen.find(&v) != seen.end()) return 0; - seen.insert(&v); + if (!seen.insert(&v).second) return 0; size_t sz = sizeof(Value); @@ -1890,8 +1886,7 @@ size_t valueSize(Value & v) sz += doString(v.path); break; case tAttrs: - if (seen.find(v.attrs) == seen.end()) { - seen.insert(v.attrs); + if (seen.insert(v.attrs).second) { sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); for (auto & i : *v.attrs) sz += doValue(*i.value); @@ -1900,8 +1895,7 @@ size_t valueSize(Value & v) case tList1: case tList2: case tListN: - if (seen.find(v.listElems()) == seen.end()) { - seen.insert(v.listElems()); + if (seen.insert(v.listElems()).second) { sz += v.listSize() * sizeof(Value *); for (size_t n = 0; n < v.listSize(); ++n) sz += doValue(*v.listElems()[n]); @@ -1922,8 +1916,7 @@ size_t valueSize(Value & v) sz += doValue(*v.primOpApp.right); break; case tExternal: - if (seen.find(v.external) != seen.end()) break; - seen.insert(v.external); + if (!seen.insert(v.external).second) break; sz += v.external->valueSize(seen); break; default: @@ -1934,8 +1927,7 @@ size_t valueSize(Value & v) }; doEnv = [&](Env & env) -> size_t { - if (seen.find(&env) != seen.end()) return 0; - seen.insert(&env); + if (!seen.insert(&env).second) return 0; size_t sz = sizeof(Env) + sizeof(Value *) * env.size; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 21a4d7917..4e22a1d77 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -277,8 +277,7 @@ static bool getDerivation(EvalState & state, Value & v, /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); + if (!done.insert(v.attrs).second) return false; DrvInfo drv(state, attrPath, v.attrs); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 967c88d9b..1dcd532ad 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -138,11 +138,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { - if (formals->argNames.find(formal.name) != formals->argNames.end()) + if (!formals->argNames.insert(formal.name).second) throw ParseError(format("duplicate formal function argument '%1%' at %2%") % formal.name % pos); formals->formals.push_front(formal); - formals->argNames.insert(formal.name); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d4c60f870..deaffa605 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -396,8 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar throw EvalError(format("attribute 'key' required, at %1%") % pos); state.forceValue(*key->value); - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); + if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ @@ -1273,13 +1272,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { + if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 00b1918a8..1f0b1541d 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "derivation", xmlAttrs); - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); + if (drvPath != "" && drvsSeen.insert(drvPath).second) showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else + else doc.writeEmptyElement("repeated"); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9a6729f9e..0e0f8a545 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -568,10 +568,9 @@ UserLock::UserLock() { auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(fnUserLock)) + if (!lockedPaths->insert(fnUserLock).second) /* We already have a lock on this one. */ continue; - lockedPaths->insert(fnUserLock); } try { @@ -620,8 +619,8 @@ UserLock::UserLock() UserLock::~UserLock() { auto lockedPaths(lockedPaths_.lock()); - assert(lockedPaths->count(fnUserLock)); - lockedPaths->erase(fnUserLock); + auto erased = lockedPaths->erase(fnUserLock); + assert(erased); } @@ -1125,10 +1124,8 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) needRestart = true; } else for (auto & i : outputs) - if (wantedOutputs.find(i) == wantedOutputs.end()) { - wantedOutputs.insert(i); + if (wantedOutputs.insert(i).second) needRestart = true; - } } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 74e706664..096593886 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -123,8 +123,7 @@ static Path out; static void addPkg(const Path & pkgDir, int priority) { - if (done.count(pkgDir)) return; - done.insert(pkgDir); + if (!done.insert(pkgDir).second) return; createLinks(pkgDir, out, priority); try { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a2af51d0e..859094e61 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1292,8 +1292,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, { checkInterrupt(); - if (done.find(path) != done.end()) return; - done.insert(path); + if (!done.insert(path).second) return; if (!isStorePath(path)) { printError(format("path '%1%' is not in the Nix store") % path); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index dddf13430..05b93d4c9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -29,8 +29,7 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (state->paths.count(path)) return; - state->paths.insert(path); + if (!state->paths.insert(path).second) return; state->pending++; } @@ -175,8 +174,7 @@ void Store::queryMissing(const PathSet & targets, { auto state(state_.lock()); - if (state->done.count(path)) return; - state->done.insert(path); + if (!state->done.insert(path).second) return; } DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); @@ -252,8 +250,7 @@ Paths Store::topoSortPaths(const PathSet & paths) if (parents.find(path) != parents.end()) throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); - if (visited.find(path) != visited.end()) return; - visited.insert(path); + if (!visited.insert(path).second) return; parents.insert(path); PathSet references; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 5b7eb1f84..0dcc264c3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -36,11 +36,10 @@ static void search(const unsigned char * s, size_t len, } if (!match) continue; string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { + if (hashes.erase(ref)) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i); seen.insert(ref); - hashes.erase(ref); } ++i; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f63c53b5..be13fa49a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -954,8 +954,7 @@ std::list> getDefaultSubstituters() StringSet done; auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); + if (!done.insert(uri).second) return; try { stores.push_back(openStore(uri)); } catch (Error & e) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 87b2e43f0..48686ce72 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -124,11 +124,10 @@ static void getAllExprs(EvalState & state, string attrName = i; if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { + if (!attrs.insert(attrName).second) { printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); continue; } - attrs.insert(attrName); /* Load the expression on demand. */ Value & vFun = state.getBuiltin("import"); Value & vArg(*state.allocValue()); @@ -307,10 +306,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, /* Insert only those elements in the final list that we haven't inserted before. */ for (auto & j : matches) - if (done.find(j.second) == done.end()) { - done.insert(j.second); + if (done.insert(j.second).second) elems.push_back(j.first); - } } checkSelectorUse(selectors); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index abdfa5e58..d448654fe 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -71,9 +71,7 @@ void printClosure(const Path & nePath, const StoreExpr & fs) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - + if (doneSet.insert(path).second) { ClosureElems::const_iterator elem = fs.closure.elems.find(path); if (elem == fs.closure.elems.end()) throw Error(format("bad closure, missing path '%1%'") % path); @@ -104,8 +102,7 @@ void printDotGraph(ref store, const PathSet & roots) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + if (!doneSet.insert(path).second) continue; cout << makeNode(path, symbolicName(path), "#ff0000"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 0cbceb02f..19384499d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -242,11 +242,10 @@ const string treeNull = " "; static void printTree(const Path & path, const string & firstPad, const string & tailPad, PathSet & done) { - if (done.find(path) != done.end()) { + if (!done.insert(path).second) { cout << format("%1%%2% [...]\n") % firstPad % path; return; } - done.insert(path); cout << format("%1%%2%\n") % firstPad % path; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 7ef571561..2e0d5cffe 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -118,8 +118,7 @@ struct CmdVerify : StorePathsCommand auto doSigs = [&](StringSet sigs) { for (auto sig : sigs) { - if (sigsSeen.count(sig)) continue; - sigsSeen.insert(sig); + if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index e02cfc03e..45e8b6796 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -117,9 +117,7 @@ Path resolveSymlink(const Path & path) std::set resolveTree(const Path & path, PathSet & deps) { std::set results; - if (deps.count(path)) - return {}; - deps.insert(path); + if (!deps.insert(path).second) return {}; for (auto & lib : runResolver(path)) { results.insert(lib); for (auto & p : resolveTree(lib, deps)) { From 926d3e5bb0ca696ab59352b25d0a818707a28186 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 22:57:37 +0200 Subject: [PATCH 199/433] Fix Bison 2.4 warning --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1dcd532ad..93834bec6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,5 +1,5 @@ %glr-parser -%pure-parser +%define api.pure %locations %define parse.error verbose %defines From 55bba8e4f51758e6c0556b24743e6946c96f6d92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:04:11 +0200 Subject: [PATCH 200/433] Make std::uncaught_exception warning less noisy --- src/libutil/json.cc | 5 +++++ src/libutil/json.hh | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 0a6fb65f0..74e37b4c4 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -171,4 +171,9 @@ JSONObject JSONPlaceholder::object() return JSONObject(state); } +JSONPlaceholder::~JSONPlaceholder() +{ + assert(!first || std::uncaught_exception()); +} + } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..83213ca66 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -168,10 +168,7 @@ public: { } - ~JSONPlaceholder() - { - assert(!first || std::uncaught_exception()); - } + ~JSONPlaceholder(); template void write(const T & v) From 4331eeb13d241dfe2d2e6a01c53915c556cac94f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:25:06 +0200 Subject: [PATCH 201/433] Filter ANSI escape sequences in -L output Otherwise, builds like NixOS VM tests may leave the terminal in a weird state and do resets. --- src/nix/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index c0bcfb0c9..5c05d6b22 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -120,7 +120,7 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { if (state.active) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { auto s2 = s + ANSI_NORMAL "\n"; From 5a303093dcae1e5ce9212616ef18f2ca51020b0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:01:21 +0200 Subject: [PATCH 202/433] Remove world-writability from per-user directories 'nix-daemon' now creates subdirectories for users when they first connect. Fixes #509 (CVE-2019-17365). Should also fix #3127. --- nix.spec.in | 2 +- scripts/install-multi-user.sh | 9 ++++----- scripts/nix-profile-daemon.sh.in | 13 ------------- scripts/nix-profile.sh.in | 14 -------------- src/libstore/local-store.cc | 24 ++++++++++++++++++++---- src/libstore/local-store.hh | 2 ++ src/libstore/store-api.hh | 3 +++ src/nix-daemon/nix-daemon.cc | 9 ++++++--- tests/nix-profile.sh | 2 -- tests/remote-store.sh | 4 ++++ tests/user-envs.sh | 2 +- 11 files changed, 41 insertions(+), 43 deletions(-) diff --git a/nix.spec.in b/nix.spec.in index 477768c6a..6b9e37637 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -106,7 +106,7 @@ chmod 1775 $RPM_BUILD_ROOT/nix/store for d in profiles gcroots; do mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user - chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user + chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user done # fix permission of nix profile diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index fd0f54184..258469a0b 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -437,16 +437,15 @@ create_build_users() { } create_directories() { + # FIXME: remove all of this because it duplicates LocalStore::LocalStore(). + _sudo "to make the basic directory structure of Nix (part 1)" \ - mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} + mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user _sudo "to make the basic directory structure of Nix (part 2)" \ - mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user - - _sudo "to make the basic directory structure of Nix (part 3)" \ mkdir -pv -m 1775 /nix/store - _sudo "to make the basic directory structure of Nix (part 4)" \ + _sudo "to make the basic directory structure of Nix (part 3)" \ chgrp "$NIX_BUILD_GROUP_NAME" /nix/store _sudo "to set up the root user's profile (part 1)" \ diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 23da5e855..3e138ac42 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,12 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" -# Set up the per-user profile. -mkdir -m 0755 -p $NIX_USER_PROFILE_DIR -if ! test -O "$NIX_USER_PROFILE_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2 -fi - if test -w $HOME; then if ! test -L $HOME/.nix-profile; then if test "$USER" != root; then @@ -26,13 +20,6 @@ if test -w $HOME; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi - # Create the per-user garbage collector roots directory. - NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER - mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR - if ! test -O "$NIX_USER_GCROOTS_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2 - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 85f1d6e5d..7f9b5877a 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -9,12 +9,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" - - if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 - fi - if [ -w "$HOME" ]; then if ! [ -L "$NIX_LINK" ]; then echo "Nix: creating $NIX_LINK" >&2 @@ -33,14 +27,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" fi - # Create the per-user garbage collector roots directory. - __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" - mkdir -m 0755 -p "$__user_gcroots" - if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 - fi - unset __user_gcroots - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 859094e61..685d43c9c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -74,10 +74,11 @@ LocalStore::LocalStore(const Params & params) multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - Path perUserDir = profilesDir + "/per-user"; - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 01777) == -1) - throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir); + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } mode_t perm = 01775; @@ -1432,4 +1433,19 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } +void LocalStore::createUser(const std::string & userName, uid_t userId) +{ + for (auto & dir : { + fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName) + }) { + createDirs(dir); + if (chmod(dir.c_str(), 0700) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, -1) == -1) + throw SysError("changing owner of directory '%s'", dir); + } +} + + } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3ae34c403..379a06af8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -293,6 +293,8 @@ private: Path getRealStoreDir() override { return realStoreDir; } + void createUser(const std::string & userName, uid_t userId) override; + friend class DerivationGoal; friend class SubstitutionGoal; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7fb568602..ba8990755 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -628,6 +628,9 @@ public: return storePath; } + virtual void createUser(const std::string & userName, uid_t userId) + { } + protected: Stats stats; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index e88aaf636..cd18489b0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -742,7 +742,8 @@ static void performOp(TunnelLogger * logger, ref store, } -static void processConnection(bool trusted) +static void processConnection(bool trusted, + const std::string & userName, uid_t userId) { MonitorFdHup monitor(from.fd); @@ -793,6 +794,8 @@ static void processConnection(bool trusted) params["path-info-cache-size"] = "0"; auto store = openStore(settings.storeUri, params); + store->createUser(userName, userId); + tunnelLogger->stopWork(); to.flush(); @@ -1053,7 +1056,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ from.fd = remote.get(); to.fd = remote.get(); - processConnection(trusted); + processConnection(trusted, user, peer.uid); exit(0); }, options); @@ -1133,7 +1136,7 @@ static int _main(int argc, char * * argv) } } } else { - processConnection(true); + processConnection(true, "root", 0); } } else { daemonLoop(argv); diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b244815e2..5b17fe3fa 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -10,5 +10,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency [ -L $TEST_HOME/.nix-profile ] [ -e $TEST_HOME/.nix-channels ] -[ -e $TEST_ROOT/profile-var/nix/gcroots/per-user/$user ] -[ -e $TEST_ROOT/profile-var/nix/profiles/per-user/$user ] diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f2f2806d0..77437658e 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -13,3 +13,7 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2 nix-store --gc --max-freed 1K killDaemon + +user=$(whoami) +[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] +[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/tests/user-envs.sh b/tests/user-envs.sh index ba6392311..aebf6a2a2 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -20,7 +20,7 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1 # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly -rm -f $HOME/.nix-defexpr +rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr nix-env -qa '*' --description | grep -q silly From c43d9f6131102f2761f22b1ec26f345d357f169c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:05:42 +0200 Subject: [PATCH 203/433] Remove some redundant initialization --- scripts/install-multi-user.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 258469a0b..40013eaaf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -448,12 +448,6 @@ create_directories() { _sudo "to make the basic directory structure of Nix (part 3)" \ chgrp "$NIX_BUILD_GROUP_NAME" /nix/store - _sudo "to set up the root user's profile (part 1)" \ - mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root - - _sudo "to set up the root user's profile (part 2)" \ - mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr" - _sudo "to place the default nix daemon configuration (part 1)" \ mkdir -pv -m 0555 /etc/nix } From 26762ceb8629af95300c0cc8c372a99282060dc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:30:31 +0200 Subject: [PATCH 204/433] nix-profile.sh: Don't create .nix-channels This is already done by the installer, so no need to do it again. --- scripts/nix-profile-daemon.sh.in | 5 ----- scripts/nix-profile.sh.in | 5 ----- tests/nix-profile.sh | 1 - 3 files changed, 11 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 3e138ac42..2133a6d3b 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -15,11 +15,6 @@ if test -w $HOME; then fi fi - # Subscribe the root user to the NixOS channel by default. - if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 7f9b5877a..8e70c68b2 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -22,11 +22,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi fi - # Subscribe the user to the unstable Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" - fi - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 5b17fe3fa..b808cf1b4 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -9,4 +9,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency [ -L $TEST_HOME/.nix-profile ] -[ -e $TEST_HOME/.nix-channels ] From 9348f9291e5d9e4ba3c4347ea1b235640f54fd79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:21:07 +0200 Subject: [PATCH 205/433] nix-env: Create ~/.nix-profile automatically --- scripts/nix-profile-daemon.sh.in | 9 --------- scripts/nix-profile.sh.in | 12 ------------ src/libstore/local-store.cc | 15 ++++++++------- src/libutil/util.cc | 10 ++++++++++ src/libutil/util.hh | 2 ++ src/nix-channel/nix-channel.cc | 8 +------- src/nix-env/nix-env.cc | 11 ++++++++--- tests/nix-channel.sh | 4 ++-- tests/nix-profile.sh | 2 -- 9 files changed, 31 insertions(+), 42 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 2133a6d3b..aa78e6fcc 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -6,15 +6,6 @@ export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" if test -w $HOME; then - if ! test -L $HOME/.nix-profile; then - if test "$USER" != root; then - ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile - fi - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 8e70c68b2..087f6ee57 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -10,18 +10,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER if [ -w "$HOME" ]; then - if ! [ -L "$NIX_LINK" ]; then - echo "Nix: creating $NIX_LINK" >&2 - if [ "$USER" != root ]; then - if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then - echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 - fi - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" - fi - fi - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 685d43c9c..2bc04da46 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -70,16 +70,17 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); - } - mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1b7449991..6f3bf7ae8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -475,6 +475,16 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + static Lazy getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 07c3d28ff..f057fdb2c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -126,6 +126,8 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +std::string getUserName(); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 06eb3d23b..70aa5c966 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -159,13 +159,7 @@ static int _main(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - ; - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER", ""); - if (name.empty()) - throw Error("cannot figure out user name"); - profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; - createDirs(dirOf(profile)); + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); enum { cNone, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 48686ce72..1b412f0bd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1422,9 +1422,14 @@ static int _main(int argc, char * * argv) if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; - globals.profile = pathExists(profileLink) - ? absPath(readLink(profileLink), dirOf(profileLink)) - : canonPath(settings.nixStateDir + "/profiles/default"); + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); } op(globals, opFlags, opArgs); diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 55f1695c4..93f837bef 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -36,7 +36,7 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles rm -f $TEST_HOME/.nix-channels @@ -55,5 +55,5 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b808cf1b4..e2e0d1090 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -7,5 +7,3 @@ rm -rf $TEST_HOME $TEST_ROOT/profile-var mkdir -p $TEST_HOME USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency - -[ -L $TEST_HOME/.nix-profile ] From 61a6176acaa8522cbcf091a34a663ef45307fef7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:38:01 +0200 Subject: [PATCH 206/433] nix-profile.sh: Remove coreutils dependency --- scripts/nix-profile.sh.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 087f6ee57..ed4d1a639 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,6 +1,4 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then - __savedpath="$PATH" - export PATH=@coreutils@ # Set up the per-user profile. # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix @@ -47,6 +45,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi - export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK NIX_USER_PROFILE_DIR fi From c9159f86cc9a2fc07e2ab1217c2d8a8824123df4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:50:46 +0200 Subject: [PATCH 207/433] nix-env: Create ~/.nix-defexpr automatically --- scripts/nix-profile-daemon.sh.in | 12 ------------ scripts/nix-profile.sh.in | 11 ----------- src/nix-env/nix-env.cc | 19 ++++++++++++------- 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index aa78e6fcc..47655080a 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,18 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" -if test -w $HOME; then - # Set up a default Nix expression from which to install stuff. - if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then - rm -f $HOME/.nix-defexpr - mkdir -p $HOME/.nix-defexpr - if [ "$USER" != root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root - fi - fi -fi - - # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index ed4d1a639..e15f7cd46 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -7,17 +7,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - if [ -w "$HOME" ]; then - # Set up a default Nix expression from which to install stuff. - __nix_defexpr="$HOME"/.nix-defexpr - [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" - mkdir -m 0755 -p "$__nix_defexpr" - if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root - fi - unset __nix_defexpr - fi - # Append ~/.nix-defexpr/channels to $NIX_PATH so that # paths work when the user has fetched the Nixpkgs channel. export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1b412f0bd..2aeb8ebb0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -192,12 +192,6 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static Path getDefNixExprPath() -{ - return getHome() + "/.nix-defexpr"; -} - - static long getPriority(EvalState & state, DrvInfo & drv) { return drv.queryMetaInt("priority", 0); @@ -1327,9 +1321,20 @@ static int _main(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getDefNixExprPath(); + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; + if (!pathExists(globals.instSource.nixExprPath)) { + createDirs(globals.instSource.nixExprPath); + replaceSymlink( + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } + globals.dryRun = false; globals.preserveInstalled = false; globals.removeAll = false; From d7bae5680fc26303acb9a9ee1a202f537841a624 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:55:05 +0200 Subject: [PATCH 208/433] Go back to 755 permission on per-user directories 700 is pointless since the store is world-readable anyway. And per-user/root/channels must be world-readable. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2bc04da46..fbf5dc35c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1441,7 +1441,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) fmt("%s/gcroots/per-user/%s", stateDir, userName) }) { createDirs(dir); - if (chmod(dir.c_str(), 0700) == -1) + if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); if (chown(dir.c_str(), userId, -1) == -1) throw SysError("changing owner of directory '%s'", dir); From 9277e72cb0aac72100c01334fdf25ea79d19052e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 21:04:21 +0200 Subject: [PATCH 209/433] Typo --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 40013eaaf..d060e5165 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -490,7 +490,7 @@ EOF We will: - make sure your computer doesn't already have Nix files - (if it does, I will tell you how to clean them up.) + (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users we'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - install Nix in to $NIX_ROOT From 20eec802ff11dd2b152715cd5c81b756d318219d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:33:28 +0200 Subject: [PATCH 210/433] Force per-user group to a known value --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fbf5dc35c..00f3a31a2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) createDirs(dir); if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, -1) == -1) + if (chown(dir.c_str(), userId, 0) == -1) throw SysError("changing owner of directory '%s'", dir); } } From bda64a2b0f79346012332ed18f5a76388e6d9bae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 00:12:30 +0200 Subject: [PATCH 211/433] Doh https://hydra.nixos.org/build/102803044 --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 00f3a31a2..d86f7b1d2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) createDirs(dir); if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, 0) == -1) + if (chown(dir.c_str(), userId, getgid()) == -1) throw SysError("changing owner of directory '%s'", dir); } } From c3aaf3b8da1a925c569389f13a861816a781a3c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 09:14:05 +0200 Subject: [PATCH 212/433] nix-env: Ignore failures creating ~/.nix-profile and ~/.nix-defexpr https://hydra.nixos.org/build/102803093 --- src/nix-env/nix-env.cc | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 2aeb8ebb0..199dc92aa 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1325,14 +1325,16 @@ static int _main(int argc, char * * argv) globals.instSource.systemFilter = "*"; if (!pathExists(globals.instSource.nixExprPath)) { - createDirs(globals.instSource.nixExprPath); - replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), - globals.instSource.nixExprPath + "/channels"); - if (getuid() != 0) + try { + createDirs(globals.instSource.nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), - globals.instSource.nixExprPath + "/channels_root"); + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error &) { } } globals.dryRun = false; @@ -1427,14 +1429,18 @@ static int _main(int argc, char * * argv) if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + globals.profile = profileLink; } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); } op(globals, opFlags, opArgs); From 95cf23ee7c5b0fd69b21811989a5668f4261fd51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 15:03:01 +0200 Subject: [PATCH 213/433] nix verify: Fix uninitialized variable --- src/nix/verify.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2e0d5cffe..74d9673b6 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -13,7 +13,7 @@ struct CmdVerify : StorePathsCommand bool noContents = false; bool noTrust = false; Strings substituterUris; - size_t sigsNeeded; + size_t sigsNeeded = 0; CmdVerify() { @@ -113,7 +113,7 @@ struct CmdVerify : StorePathsCommand else { StringSet sigsSeen; - size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { From 906d56a96b442d4dd8f924c1ce0d1eec0e214af3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Oct 2019 18:48:59 +0200 Subject: [PATCH 214/433] ssh-ng: Don't set CPU affinity on the remote Fixes #3138. --- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 5 +++++ src/libstore/ssh-store.cc | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7f202a6b..f34369d8f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -151,7 +151,7 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) conn.to << 1 << cpu; else diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 82fbec092..1f375dd71 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: const Setting maxConnectionAge{(Store*) this, std::numeric_limits::max(), "max-connection-age", "number of seconds to reuse a connection"}; + virtual bool sameMachine() = 0; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -146,6 +148,9 @@ public: std::string getUri() override; + bool sameMachine() + { return true; } + private: ref openConnection() override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 39205ae2c..93e117389 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -35,6 +35,9 @@ public: return uriScheme + host; } + bool sameMachine() + { return false; } + void narFromPath(const Path & path, Sink & sink) override; ref getFSAccessor() override; From 499b0388759db0f9f385da402a4bba551268aa99 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 12 Oct 2019 19:02:57 -0400 Subject: [PATCH 215/433] Fix sandbox fallback settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tmpDirInSandbox is different when in sandboxed vs. non-sandboxed. Since we don’t know ahead of time here whether sandboxing is enabled, we need to reset all of the env vars we’ve set previously. This fixes the issue encountered in https://github.com/NixOS/nixpkgs/issues/70856. --- src/libstore/build.cc | 60 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e0f8a545..273f25363 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -956,6 +956,9 @@ private: /* Fill in the environment for the builder. */ void initEnv(); + /* Setup tmp dir location. */ + void initTmpDir(); + /* Write a JSON file containing the derivation attributes. */ void writeStructuredAttrs(); @@ -2383,6 +2386,7 @@ void DerivationGoal::startBuilder() if (res != 0 && settings.sandboxFallback) { useChroot = false; tmpDirInSandbox = tmpDir; + initTmpDir(); goto fallback; } else if (res != 0) throw Error("unable to start build process"); @@ -2438,32 +2442,7 @@ void DerivationGoal::startBuilder() } -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - +void DerivationGoal::initTmpDir() { /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing @@ -2501,6 +2480,35 @@ void DerivationGoal::initEnv() inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ env["PWD"] = tmpDirInSandbox; +} + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' From 96c84937c49435525d0733437aa88902b2c8caf6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 13 Oct 2019 16:41:49 -0400 Subject: [PATCH 216/433] Move tmpDirInSandbox to initTmpDir --- src/libstore/build.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 273f25363..03a069ae2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1961,13 +1961,6 @@ void DerivationGoal::startBuilder() auto drvName = storePathToName(drvPath); tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -2385,7 +2378,6 @@ void DerivationGoal::startBuilder() int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { useChroot = false; - tmpDirInSandbox = tmpDir; initTmpDir(); goto fallback; } else if (res != 0) @@ -2443,6 +2435,14 @@ void DerivationGoal::startBuilder() void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing From 7c568d4c6e3291f01833aa9cf3779dfdcdaa4505 Mon Sep 17 00:00:00 2001 From: xbreak Date: Sat, 12 Oct 2019 13:00:19 +0000 Subject: [PATCH 217/433] Downloader: Warn if no trusted CA file has been configured --- src/libstore/download.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cdf56e09d..e5f8556b7 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -285,6 +285,7 @@ struct CurlDownloader : public Downloader } if (request.verifyTLS) { + debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); } else { From f0ec4b4ce478a8d2760203e8192275b88c770e1c Mon Sep 17 00:00:00 2001 From: Steven Shaw Date: Sat, 19 Oct 2019 13:25:18 +1000 Subject: [PATCH 218/433] Fix unset variable in installer --- scripts/install-nix-from-closure.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 35926f3da..3f1581854 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -141,11 +141,9 @@ if [ -z "$_NIX_INSTALLER_TEST" ]; then fi added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then - # Make the shell source nix.sh during login. - p=$HOME/.nix-profile/etc/profile.d/nix.sh - for i in .bash_profile .bash_login .profile; do fn="$HOME/$i" if [ -w "$fn" ]; then @@ -157,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done - fi if [ -z "$added" ]; then From d8730fb86facadbef22d3df7f8a743a56e7ed53c Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Sun, 20 Oct 2019 19:08:05 -0700 Subject: [PATCH 219/433] Document import syntax --- doc/manual/expressions/builtins.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..21da8b87b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,13 +733,21 @@ builtins.genList (x: x * x) 5 import path + import + <path> builtins.import path + builtins.import + <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. Evaluation aborts if the + in that directory is loaded. If the + <path> syntax is used, the path will be resolved + relative to those listed in the NIX_PATH environment + variable (see the section on NIX_PATH + for details on how the resolution works). Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From 389a2cebed7cd72bda524ece0a56af2888cd80b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 13:14:39 +0200 Subject: [PATCH 220/433] SourceExprCommand::getSourceExpr(): Allocate more space Fixes #3140. --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 52f9778fc..0e8bba39d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -39,7 +39,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, searchPath.size() + 1); + state.mkAttrs(*vSourceExpr, 1024); mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); From aabf5c86c9df1b4e1616a4cf06ee14a6edf2e5e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Oct 2019 17:45:09 +0200 Subject: [PATCH 221/433] Add experimental-features setting Experimental features are now opt-in. There is currently one experimental feature: "nix-command" (which enables the "nix" command. This will allow us to merge experimental features more quickly, without committing to supporting them indefinitely. Typical usage: $ nix build --experimental-features 'nix-command flakes' nixpkgs#hello (cherry picked from commit 8e478c234100cf03ea1b777d4bd42a9be7be9e8c, without the "flakes" feature) --- src/libstore/globals.cc | 7 +++++++ src/libstore/globals.hh | 5 +++++ src/nix/main.cc | 2 ++ tests/init.sh | 1 + 4 files changed, 15 insertions(+) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1c2c08715..249c36673 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +void Settings::requireExperimentalFeature(const std::string & name) +{ + auto & f = experimentalFeatures.get(); + if (std::find(f.begin(), f.end(), name) == f.end()) + throw Error("experimental Nix feature '%s' is disabled", name); +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ab1c09aa2..1221e4db7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -353,6 +353,11 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + void requireExperimentalFeature(const std::string & name); }; diff --git a/src/nix/main.cc b/src/nix/main.cc index a80fd0ea6..22f5e8c4c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -138,6 +138,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + settings.requireExperimentalFeature("nix-command"); + initPlugins(); if (!args.command) args.showHelpAndExit(); diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..6a119aad0 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,6 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < Date: Fri, 30 Mar 2018 00:56:13 +0200 Subject: [PATCH 222/433] Allow content-addressable paths to have references This adds a command 'nix make-content-addressable' that rewrites the specified store paths into content-addressable paths. The advantage of such paths is that 1) they can be imported without signatures; 2) they can enable deduplication in cases where derivation changes do not cause output changes (apart from store path hashes). For example, $ nix make-content-addressable -r nixpkgs.cowsay rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49' ... rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16' We can then copy the resulting closure to another store without signatures: $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 In order to support self-references in content-addressable paths, these paths are hashed "modulo" self-references, meaning that self-references are zeroed out during hashing. Somewhat annoyingly, this means that the NAR hash stored in the Nix database is no longer necessarily equal to the output of "nix hash-path"; for content-addressable paths, you need to pass the --modulo flag: $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 | jq -r .[].narHash sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67 0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw --- src/libstore/build.cc | 19 +------ src/libstore/local-store.cc | 21 ++++++-- src/libstore/references.cc | 62 ++++++++++++++++++++++ src/libstore/references.hh | 29 +++++++++- src/libstore/store-api.cc | 38 +++++++------ src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 20 ++----- src/libutil/hash.hh | 11 ++-- src/libutil/serialise.hh | 2 +- src/libutil/util.cc | 13 +++++ src/libutil/util.hh | 14 +++++ src/nix/command.cc | 2 +- src/nix/command.hh | 4 ++ src/nix/hash.cc | 22 +++++++- src/nix/make-content-addressable.cc | 82 +++++++++++++++++++++++++++++ src/nix/verify.cc | 12 +++-- 16 files changed, 289 insertions(+), 65 deletions(-) create mode 100644 src/nix/make-content-addressable.cc diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e0f8a545..68c2f2ce3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -727,23 +727,6 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map StringRewrites; - - -std::string rewriteStrings(std::string s, const StringRewrites & rewrites) -{ - for (auto & i : rewrites) { - size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) - s.replace(j, i.first.size(), i.second); - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -865,7 +848,7 @@ private: #endif /* Hash rewriting. */ - StringRewrites inputRewrites, outputRewrites; + StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d86f7b1d2..1280ecab5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,6 +5,7 @@ #include "worker-protocol.hh" #include "derivations.hh" #include "nar-info.hh" +#include "references.hh" #include #include @@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + std::unique_ptr hashSink; + if (info.ca == "") + hashSink = std::make_unique(htSHA256); + else + hashSink = std::make_unique(htSHA256, storePathToHash(info.path)); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); - hashSink(data, n); + (*hashSink)(data, n); return n; }); restorePath(realPath, wrapperSource); - auto hashResult = hashSink.finish(); + auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of '%1%'") % i); - HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + std::unique_ptr hashSink; + if (info->ca == "") + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(info->path)); + + dumpPath(toRealPath(i), *hashSink); + auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError(format("path '%1%' was modified! " diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 0dcc264c3..605ca9815 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path, } +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : from(from), to(to), nextSink(nextSink) +{ + assert(from.size() == to.size()); +} + +void RewritingSink::operator () (const unsigned char * data, size_t len) +{ + std::string s(prev); + s.append((const char *) data, len); + + size_t j = 0; + while ((j = s.find(from, j)) != string::npos) { + matches.push_back(pos + j); + s.replace(j, from.size(), to); + } + + prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + + auto consumed = s.size() - prev.size(); + + pos += consumed; + + if (consumed) nextSink((unsigned char *) s.data(), consumed); +} + +void RewritingSink::flush() +{ + if (prev.empty()) return; + pos += prev.size(); + nextSink((unsigned char *) prev.data(), prev.size()); + prev.clear(); +} + +HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) + : hashSink(ht) + , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) +{ +} + +void HashModuloSink::operator () (const unsigned char * data, size_t len) +{ + rewritingSink(data, len); +} + +HashResult HashModuloSink::finish() +{ + rewritingSink.flush(); + + /* Hash the positions of the self-references. This ensures that a + NAR with self-references and a NAR with some of the + self-references already zeroed out do not produce a hash + collision. FIXME: proof. */ + for (auto & pos : rewritingSink.matches) { + auto s = fmt("|%d", pos); + hashSink((unsigned char *) s.data(), s.size()); + } + + auto h = hashSink.finish(); + return {h.first, rewritingSink.pos}; +} + } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 013809d12..c38bdd720 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,5 +7,32 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); - + +struct RewritingSink : Sink +{ + std::string from, to, prev; + Sink & nextSink; + uint64_t pos = 0; + + std::vector matches; + + RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + + void operator () (const unsigned char * data, size_t len) override; + + void flush(); +}; + +struct HashModuloSink : AbstractHashSink +{ + HashSink hashSink; + RewritingSink rewritingSink; + + HashModuloSink(HashType ht, const std::string & modulus); + + void operator () (const unsigned char * data, size_t len) override; + + HashResult finish() override; +}; + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index be13fa49a..1a1b24e3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id, } -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const +static std::string makeType(string && type, const PathSet & references) { - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, + for (auto & i : references) { + type += ":"; + type += i; + } + return type; +} + + +Path Store::makeFixedOutputPath(bool recursive, + const Hash & hash, const string & name, const PathSet & references) const +{ + if (hash.type == htSHA256 && recursive) { + return makeStorePath(makeType("source", references), hash, name); + } else { + assert(references.empty()); + return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), - name); + hash.to_string(Base16) + ":"), name); + } } @@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash, /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); + return makeStorePath(makeType("text", references), hash, name); } @@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - if (references.empty() && - store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + auto refs = references; + replaceInSet(refs, path, std::string("self")); + if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path) return true; else warn(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ba8990755..250e1a4bc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -303,7 +303,8 @@ public: const Hash & hash, const string & name) const; Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const; + const Hash & hash, const string & name, + const PathSet & references = {}) const; Path makeTextPath(const string & name, const Hash & hash, const PathSet & references) const; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1c14ebb18..7caee1da7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s) Hash hashFile(HashType ht, const Path & path) { - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); - - std::vector buf(8192); - ssize_t n; - while ((n = read(fd.get(), buf.data(), buf.size()))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file '%1%'") % path); - update(ht, ctx, buf.data(), n); - } - - finish(ht, ctx, hash.hash); - return hash; + HashSink sink(ht); + readFile(path, sink); + return sink.finish().first; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2dbc3b630..ffa43ecf5 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -111,7 +111,12 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public BufferedSink +struct AbstractHashSink : virtual Sink +{ + virtual HashResult finish() = 0; +}; + +class HashSink : public BufferedSink, public AbstractHashSink { private: HashType ht; @@ -122,8 +127,8 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); + void write(const unsigned char * data, size_t len) override; + HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a344a5ac7..0120aeecb 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct Sink /* A buffered abstract sink. */ -struct BufferedSink : Sink +struct BufferedSink : virtual Sink { size_t bufSize, bufPos; std::unique_ptr buffer; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6f3bf7ae8..2e416edef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s, } +std::string rewriteStrings(const std::string & _s, const StringMap & rewrites) +{ + auto s = _s; + for (auto & i : rewrites) { + if (i.first == i.second) continue; + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f057fdb2c..3493e80b5 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -362,6 +362,20 @@ string replaceStrings(const std::string & s, const std::string & from, const std::string & to); +std::string rewriteStrings(const std::string & s, const StringMap & rewrites); + + +/* If a set contains 'from', remove it and insert 'to'. */ +template +void replaceInSet(std::set & set, const T & from, const T & to) +{ + auto i = set.find(from); + if (i == set.end()) return; + set.erase(i); + set.insert(to); +} + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); diff --git a/src/nix/command.cc b/src/nix/command.cc index 3d7d582d6..532f331a7 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -129,7 +129,7 @@ void StorePathsCommand::run(ref store) } else { - for (auto & p : toStorePaths(store, NoBuild, installables)) + for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p); if (recursive) { diff --git a/src/nix/command.hh b/src/nix/command.hh index 97a6fee7f..45ad1cd2a 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -139,6 +139,10 @@ private: bool recursive = false; bool all = false; +protected: + + RealiseMode realiseMode = NoBuild; + public: StorePathsCommand(bool recursive = false); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index af4105e28..9c06e6116 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -2,6 +2,8 @@ #include "hash.hh" #include "legacy.hh" #include "shared.hh" +#include "references.hh" +#include "archive.hh" using namespace nix; @@ -13,6 +15,7 @@ struct CmdHash : Command bool truncate = false; HashType ht = htSHA256; std::vector paths; + std::experimental::optional modulus; CmdHash(Mode mode) : mode(mode) { @@ -23,6 +26,11 @@ struct CmdHash : Command mkFlag() .longName("type") .mkHashTypeFlag(&ht); + mkFlag() + .longName("modulo") + .description("compute hash modulo specified string") + .labels({"modulus"}) + .dest(&modulus); expectArgs("paths", &paths); } @@ -41,7 +49,19 @@ struct CmdHash : Command void run() override { for (auto path : paths) { - Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + + std::unique_ptr hashSink; + if (modulus) + hashSink = std::make_unique(ht, *modulus); + else + hashSink = std::make_unique(ht); + + if (mode == mFile) + readFile(path, *hashSink); + else + dumpPath(path, *hashSink); + + Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % h.to_string(base, base == SRI); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc new file mode 100644 index 000000000..14ed06413 --- /dev/null +++ b/src/nix/make-content-addressable.cc @@ -0,0 +1,82 @@ +#include "command.hh" +#include "store-api.hh" +#include "references.hh" + +using namespace nix; + +struct CmdMakeContentAddressable : StorePathsCommand +{ + CmdMakeContentAddressable() + { + realiseMode = Build; + } + + std::string name() override + { + return "make-content-addressable"; + } + + std::string description() override + { + return "test"; + } + + void run(ref store, Paths storePaths) override + { + auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); + + paths.reverse(); + + std::map remappings; + + for (auto & path : paths) { + auto oldInfo = store->queryPathInfo(path); + auto oldHashPart = storePathToHash(path); + auto name = storePathToName(path); + + StringSink sink; + store->narFromPath(path, sink); + + StringMap rewrites; + + ValidPathInfo info; + for (auto & ref : oldInfo->references) { + if (ref == path) + info.references.insert("self"); + else { + auto replacement = get(remappings, ref, ref); + // FIXME: warn about unremapped paths? + info.references.insert(replacement); + if (replacement != ref) + rewrites[storePathToHash(ref)] = storePathToHash(replacement); + } + } + + *sink.s = rewriteStrings(*sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + + info.narHash = hashModuloSink.finish().first; + info.narSize = sink.s->size(); + replaceInSet(info.references, path, std::string("self")); + info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references); + replaceInSet(info.references, std::string("self"), info.path); + info.ca = makeFixedOutputCA(true, info.narHash); + + printError("rewrote '%s' to '%s'", path, info.path); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + + store->addToStore(info, *source); + + remappings[path] = info.path; + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 74d9673b6..4b0f80c62 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" +#include "references.hh" #include @@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - HashSink sink(info->narHash.type); - store->narFromPath(info->path, sink); + std::unique_ptr hashSink; + if (info->ca == "") + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(info->path)); - auto hash = sink.finish(); + store->narFromPath(info->path, *hashSink); + + auto hash = hashSink->finish(); if (hash.first != info->narHash) { corrupted++; From d77970fde7fac66cf8c5cdb7688906ba2f5d5362 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:49:16 +0200 Subject: [PATCH 223/433] Fix build --- src/nix/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9c06e6116..53a9fd204 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -15,7 +15,7 @@ struct CmdHash : Command bool truncate = false; HashType ht = htSHA256; std::vector paths; - std::experimental::optional modulus; + std::optional modulus; CmdHash(Mode mode) : mode(mode) { From e68736936a11c76725915e628b8d41d9351bbcf2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:58:17 +0200 Subject: [PATCH 224/433] nix make-content-addressable: Add examples --- src/nix/make-content-addressable.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 14ed06413..16344ee14 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -18,9 +18,22 @@ struct CmdMakeContentAddressable : StorePathsCommand std::string description() override { - return "test"; + return "rewrite a path or closure to content-addressable form"; } + Examples examples() override + { + return { + Example{ + "To create a content-addressable representation of GNU Hello (but not its dependencies):", + "nix make-content-addressable nixpkgs.hello" + }, + Example{ + "To compute a content-addressable representation of the current NixOS system closure:", + "nix make-content-addressable -r /run/current-system" + }, + }; + } void run(ref store, Paths storePaths) override { auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); From 629b9b0049363e091b76b7f60a8357d9f94733cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 18:05:31 +0200 Subject: [PATCH 225/433] Mark content-addressable paths with references as experimental --- src/libstore/local-store.cc | 5 ++++- src/nix/hash.cc | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1280ecab5..49061f27d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1013,8 +1013,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, std::unique_ptr hashSink; if (info.ca == "") hashSink = std::make_unique(htSHA256); - else + else { + if (!info.references.empty()) + settings.requireExperimentalFeature("ca-references"); hashSink = std::make_unique(htSHA256, storePathToHash(info.path)); + } LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 53a9fd204..d7451376c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -26,11 +26,13 @@ struct CmdHash : Command mkFlag() .longName("type") .mkHashTypeFlag(&ht); + #if 0 mkFlag() .longName("modulo") .description("compute hash modulo specified string") .labels({"modulus"}) .dest(&modulus); + #endif expectArgs("paths", &paths); } From 9be7787ec04d604a4e65a22a6b9ae56567aaf618 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 13:12:41 -0700 Subject: [PATCH 226/433] Revert "Document import syntax" This reverts commit d8730fb86facadbef22d3df7f8a743a56e7ed53c. --- doc/manual/expressions/builtins.xml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 21da8b87b..465fa1e0b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,21 +733,13 @@ builtins.genList (x: x * x) 5 import path - import - <path> builtins.import path - builtins.import - <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. If the - <path> syntax is used, the path will be resolved - relative to those listed in the NIX_PATH environment - variable (see the section on NIX_PATH - for details on how the resolution works). Evaluation aborts if the + in that directory is loaded. Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From b7a936224ee6e1ad9c842f618bbd5bc1864ebd54 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:11:26 -0700 Subject: [PATCH 227/433] Add hint about path in builtins.import --- doc/manual/expressions/builtins.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..8b53cd578 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -746,6 +746,10 @@ builtins.genList (x: x * x) 5 separate file, and use it from Nix expressions in other files. + Unlike some languages, import is a regular + function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + values (see ). + A Nix expression loaded by import must not contain any free variables (identifiers that are not defined in the Nix expression itself and are not From f107a270026596e525bdf8df0e2d331783ec4a10 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:16:55 -0700 Subject: [PATCH 228/433] Tweak path hint --- doc/manual/expressions/builtins.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 8b53cd578..f39b393bf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -747,7 +747,8 @@ builtins.genList (x: x * x) 5 files. Unlike some languages, import is a regular - function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + function in Nix. Paths using the angle bracket syntax (e.g., + import <foo>) are normal path values (see ). A Nix expression loaded by import must From c92ea927e508f3c06b6b3ee68e1f0565264cac2c Mon Sep 17 00:00:00 2001 From: Chaz Schlarp Date: Tue, 22 Oct 2019 16:04:49 -0700 Subject: [PATCH 229/433] Remove superfluous IAM action for S3 cache `s3:ListObjects` isn't a real IAM action, but _is_ the name of an S3 API method. `s3:ListBucket` is the relevant action for that method. https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazons3.html --- doc/manual/packages/s3-substituter.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index 1722090ef..868b5a66d 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -159,7 +159,6 @@ the S3 URL: "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts", - "s3:ListObjects", "s3:PutObject" ], "Resource": [ From 73ff84f6a820ac1c6e0fe502692432c8945fd8b0 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 23 Oct 2019 15:29:16 +0200 Subject: [PATCH 230/433] nix repl: add :edit command This allows to have a repl-centric workflow to working on nixpkgs. Usage: :edit - heuristic that find the package file path :edit - just open the editor on the file path Once invoked, `nix repl` will open $EDITOR on that file path. Once the editor exits, `nix repl` will automatically reload itself. --- src/nix/repl.cc | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index f857b2e89..79f365cdb 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "attr-path.hh" #include "store-api.hh" #include "common-eval-args.hh" #include "get-drvs.hh" @@ -440,6 +441,7 @@ bool NixRepl::processLine(string line) << " = Bind expression to variable\n" << " :a Add attributes from resulting set to scope\n" << " :b Build derivation\n" + << " :e Open the derivation in $EDITOR\n" << " :i Build derivation, then install result into current profile\n" << " :l Load Nix expression and add it to scope\n" << " :p Evaluate and print expression recursively\n" @@ -466,6 +468,61 @@ bool NixRepl::processLine(string line) reloadFiles(); } + else if (command == ":e" || command == ":edit") { + Value v; + evalString(arg, v); + + std::string filename; + int lineno = 0; + + if (v.type == tPath || v.type == tString) { + PathSet context; + filename = state.coerceToString(noPos, v, context); + lineno = 0; + } else { + // assume it's a derivation + Value * v2; + try { + auto dummyArgs = state.allocBindings(0); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); + } catch (Error &) { + throw Error("package '%s' has no source location information", arg); + } + + auto pos = state.forceString(*v2); + debug("position is %s", pos); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute '%s'", pos); + + filename = std::string(pos, 0, colon); + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument & e) { + throw Error("cannot parse line number '%s'", pos); + } + + } + + // Open in EDITOR + auto editor = getEnv("EDITOR", "cat"); + auto args = tokenizeString(editor); + if (lineno > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos)) + args.push_back(fmt("+%d", lineno)); + args.push_back(filename); + editor = args.front(); + args.pop_front(); + runProgram(editor, args); + + // Reload right after exiting the editor + state.resetFileCache(); + reloadFiles(); + } + else if (command == ":t") { Value v; evalString(arg, v); From 207a537343ace1d39ac125059456d832a5237f2e Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 23 Oct 2019 16:48:28 +0200 Subject: [PATCH 231/433] libutil: add editorFor heuristic --- src/libutil/args.cc | 13 +++++++++++++ src/libutil/args.hh | 3 +++ src/nix/edit.cc | 17 +++++------------ src/nix/repl.cc | 11 ++--------- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7af2a1bf7..35ec3e4ab 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -178,6 +178,19 @@ Strings argvToStrings(int argc, char * * argv) return args; } +Strings editorFor(std::string filename, int lineno) +{ + auto editor = getEnv("EDITOR", "cat"); + auto args = tokenizeString(editor); + if (lineno > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos)) + args.push_back(fmt("+%d", lineno)); + args.push_back(filename); + return args; +} + std::string renderLabels(const Strings & labels) { std::string res; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ad5fcca39..22702c2d8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -190,6 +190,9 @@ public: Strings argvToStrings(int argc, char * * argv); +/* Helper function to generate args that invoke $EDITOR on filename:lineno */ +Strings editorFor(std::string filename, int lineno); + /* Helper function for rendering argument labels. */ std::string renderLabels(const Strings & labels); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index a6169f118..3a27b9cca 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -59,22 +59,15 @@ struct CmdEdit : InstallableCommand throw Error("cannot parse line number '%s'", pos); } - auto editor = getEnv("EDITOR", "cat"); - - auto args = tokenizeString(editor); - - if (editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos) - args.push_back(fmt("+%d", lineno)); - - args.push_back(filename); - stopProgressBar(); + auto args = editorFor(filename, lineno); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); - throw SysError("cannot run editor '%s'", editor); + std::string command; + for (const auto &arg : args) command += " '" + arg + "'"; + throw SysError("cannot run command%s", command); } }; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 79f365cdb..d4334cf7f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -506,15 +506,8 @@ bool NixRepl::processLine(string line) } // Open in EDITOR - auto editor = getEnv("EDITOR", "cat"); - auto args = tokenizeString(editor); - if (lineno > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos)) - args.push_back(fmt("+%d", lineno)); - args.push_back(filename); - editor = args.front(); + auto args = editorFor(filename, lineno); + auto editor = args.front(); args.pop_front(); runProgram(editor, args); From 59c72497696eaafa294c34699795788d24d68c68 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 23 Oct 2019 17:21:10 +0200 Subject: [PATCH 232/433] libexpr: add findDerivationFilename extract the derivation to filename:lineno heuristic --- src/libexpr/attr-path.cc | 28 ++++++++++++++++++++++++++++ src/libexpr/attr-path.hh | 5 +++++ src/nix/edit.cc | 23 ++--------------------- src/nix/repl.cc | 23 +---------------------- 4 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index b0f80db32..7a6d8dfd0 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -93,4 +93,32 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, } +std::tuple findDerivationFilename(EvalState & state, Value & v, std::string what) +{ + Value * v2; + try { + auto dummyArgs = state.allocBindings(0); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); + } catch (Error &) { + throw Error("package '%s' has no source location information", what); + } + + auto pos = state.forceString(*v2); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute '%s'", pos); + + std::string filename(pos, 0, colon); + int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument & e) { + throw Error("cannot parse line number '%s'", pos); + } + + return std::make_tuple(filename, lineno); +} + + } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 46a341950..dca94cc78 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -4,10 +4,15 @@ #include #include +#include namespace nix { Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); +/* Heuristic to find the filename and lineno or a derivation. */ +std::tuple findDerivationFilename(EvalState & state, + Value & v, std::string what); + } diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 3a27b9cca..a4aa40bed 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -36,28 +36,9 @@ struct CmdEdit : InstallableCommand auto v = installable->toValue(*state); - Value * v2; - try { - auto dummyArgs = state->allocBindings(0); - v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); - } catch (Error &) { - throw Error("package '%s' has no source location information", installable->what()); - } - - auto pos = state->forceString(*v2); - debug("position is %s", pos); - - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); - - std::string filename(pos, 0, colon); + std::string filename; int lineno; - try { - lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); - } + std::tie(filename, lineno) = findDerivationFilename(*state, *v, installable->what()); stopProgressBar(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d4334cf7f..5cfb408f7 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -481,28 +481,7 @@ bool NixRepl::processLine(string line) lineno = 0; } else { // assume it's a derivation - Value * v2; - try { - auto dummyArgs = state.allocBindings(0); - v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); - } catch (Error &) { - throw Error("package '%s' has no source location information", arg); - } - - auto pos = state.forceString(*v2); - debug("position is %s", pos); - - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); - - filename = std::string(pos, 0, colon); - try { - lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); - } - + std::tie(filename, lineno) = findDerivationFilename(state, v, arg); } // Open in EDITOR From 2f96a89646c6e55e2f1bbb80805dcbbe60fa94ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Oct 2019 21:24:00 +0200 Subject: [PATCH 233/433] install-multi-user.sh: Remove unused variables https://hydra.nixos.org/build/104119659 --- scripts/install-multi-user.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d060e5165..13762cba3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -19,9 +19,6 @@ readonly BLUE_UL='\033[38;4;34m' readonly GREEN='\033[38;32m' readonly GREEN_UL='\033[38;4;32m' readonly RED='\033[38;31m' -readonly RED_UL='\033[38;4;31m' -readonly YELLOW='\033[38;33m' -readonly YELLOW_UL='\033[38;4;33m' readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" From 70cab0587dcbc2d0600c07c96b477a2ea2fcd6f3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Oct 2019 07:23:05 -0400 Subject: [PATCH 234/433] Switch to nixpkgs 19.09 --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index 0fe683518..8fcf04680 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index 8167f87a2..9c504f024 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; From e583df52800b4baa1564b027fe3b83a21756c2cc Mon Sep 17 00:00:00 2001 From: Robin Gloster Date: Sun, 27 Oct 2019 10:15:51 +0100 Subject: [PATCH 235/433] builtins.toJSON: fix __toString usage --- src/libexpr/eval.cc | 23 +++++++++++++++++------ src/libexpr/eval.hh | 4 ++++ src/libexpr/value-to-json.cc | 7 ++++++- tests/lang/eval-okay-tojson.exp | 2 +- tests/lang/eval-okay-tojson.nix | 1 + 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..8f728b906 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1565,6 +1565,19 @@ bool EvalState::isDerivation(Value & v) } +std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore, bool copyToStore) +{ + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { @@ -1583,13 +1596,11 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } if (v.type == tAttrs) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { - Value v1; - callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore); + auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; } - i = v.attrs->find(sOutPath); + auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..82e6ca877 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -8,6 +8,7 @@ #include "config.hh" #include +#include #include @@ -195,6 +196,9 @@ public: set with attribute `type = "derivation"'). */ bool isDerivation(Value & v); + std::optional tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore = false, bool copyToStore = true); + /* String coercion. Converts strings, paths and derivations to a string. If `coerceMore' is set, also converts nulls, integers, booleans and lists to a string. If `copyToStore' is set, diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 72e413e44..5fe8570ad 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -40,7 +40,12 @@ void printValueAsJSON(EvalState & state, bool strict, break; case tAttrs: { - Bindings::iterator i = v.attrs->find(state.sOutPath); + auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { auto obj(out.object()); StringSet names; diff --git a/tests/lang/eval-okay-tojson.exp b/tests/lang/eval-okay-tojson.exp index 33588493f..e92aae323 100644 --- a/tests/lang/eval-okay-tojson.exp +++ b/tests/lang/eval-okay-tojson.exp @@ -1 +1 @@ -"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44}" +"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44,\"k\":\"foo\"}" diff --git a/tests/lang/eval-okay-tojson.nix b/tests/lang/eval-okay-tojson.nix index c046ba4ae..ce67943be 100644 --- a/tests/lang/eval-okay-tojson.nix +++ b/tests/lang/eval-okay-tojson.nix @@ -9,4 +9,5 @@ builtins.toJSON h = [ "a" [ "b" { "foo\nbar" = {}; } ] ]; i = 1 + 2; j = 1.44; + k = { __toString = self: self.a; a = "foo"; }; } From 9d612c393abc3a73590650d24bcfe2ee57792872 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 27 Oct 2019 09:34:33 +0100 Subject: [PATCH 236/433] Add inNixShell = true to nix-shell auto-call This is an alternative to the IN_NIX_SHELL environment variable, allowing the expression to adapt itself to nix-shell without triggering those adaptations when used as a dependency of another shell. Closes #3147 --- src/nix-build/nix-build.cc | 16 +++++++++++++++- tests/nix-shell.sh | 8 ++++---- tests/shell.nix | 3 ++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 085650e4f..357986980 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -245,7 +245,21 @@ static void _main(int argc, char * * argv) auto state = std::make_unique(myArgs.searchPath, store); state->repair = repair; - Bindings & autoArgs = *myArgs.getAutoArgs(*state); + Bindings & autoArgs = *[&](){ + Bindings *userAutoArgs = myArgs.getAutoArgs(*state); + if (runEnv) { + Bindings * res = state->allocBindings(userAutoArgs->size() + 1); + Value * tru = state->allocValue(); + mkBool(*tru, true); + res->push_back(Attr(state->symbols.create("inNixShell"), tru)); + for (auto & i : *userAutoArgs) { + res->push_back(i); + } + res->sort(); + return res; + } + else return userAutoArgs; + }(); if (packages) { std::ostringstream joined; diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index ee502dddb..235e2a5ff 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -7,9 +7,9 @@ export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz export NIX_BUILD_SHELL=$SHELL output=$(nix-shell --pure shell.nix -A shellDrv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') -[ "$output" = " - foo - bar" ] +[ "$output" = " - foo - bar - true" ] # Test --keep output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ @@ -19,10 +19,10 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] # Test nix-shell on a .drv symlink diff --git a/tests/shell.nix b/tests/shell.nix index eb39f9039..6cb4f082b 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -1,4 +1,4 @@ -{ }: +{ inNixShell ? false }: with import ./config.nix; @@ -22,6 +22,7 @@ let pkgs = rec { name = "shellDrv"; builder = "/does/not/exist"; VAR_FROM_NIX = "bar"; + TEST_inNixShell = if inNixShell then "true" else "false"; inherit stdenv; }; From 3913afdd69a5bd065d6096dd569397a836ac6857 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Oct 2019 17:58:25 +0100 Subject: [PATCH 237/433] Simplification --- src/nix-build/nix-build.cc | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 357986980..1c5d71681 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -245,21 +245,17 @@ static void _main(int argc, char * * argv) auto state = std::make_unique(myArgs.searchPath, store); state->repair = repair; - Bindings & autoArgs = *[&](){ - Bindings *userAutoArgs = myArgs.getAutoArgs(*state); - if (runEnv) { - Bindings * res = state->allocBindings(userAutoArgs->size() + 1); - Value * tru = state->allocValue(); - mkBool(*tru, true); - res->push_back(Attr(state->symbols.create("inNixShell"), tru)); - for (auto & i : *userAutoArgs) { - res->push_back(i); - } - res->sort(); - return res; - } - else return userAutoArgs; - }(); + auto autoArgs = myArgs.getAutoArgs(*state); + + if (runEnv) { + auto newArgs = state->allocBindings(autoArgs->size() + 1); + auto tru = state->allocValue(); + mkBool(*tru, true); + newArgs->push_back(Attr(state->symbols.create("inNixShell"), tru)); + for (auto & i : *autoArgs) newArgs->push_back(i); + newArgs->sort(); + autoArgs = newArgs; + } if (packages) { std::ostringstream joined; @@ -313,9 +309,9 @@ static void _main(int argc, char * * argv) state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot)); state->forceValue(v); - getDerivations(*state, v, "", autoArgs, drvs, false); + getDerivations(*state, v, "", *autoArgs, drvs, false); } } From f1c0b2c0e16154e738f342f2defa42c4d9348c4c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Oct 2018 17:05:55 +0200 Subject: [PATCH 238/433] Add O(1)-memory copyPath() function --- src/libutil/archive.cc | 9 +++++++++ src/libutil/archive.hh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3aa120270..db544a212 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -375,4 +375,13 @@ void copyNAR(Source & source, Sink & sink) } +void copyPath(const Path & from, const Path & to) +{ + auto source = sinkToSource([&](Sink & sink) { + dumpPath(from, sink); + }); + restorePath(to, *source); +} + + } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 25be426c1..768fe2536 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -77,6 +77,8 @@ void restorePath(const Path & path, Source & source); /* Read a NAR from 'source' and write it to 'sink'. */ void copyNAR(Source & source, Sink & sink); +void copyPath(const Path & from, const Path & to); + extern const std::string narVersionMagic1; From f7ce80f90ab364058e28989af68ba993fd8180a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 11:22:13 +0200 Subject: [PATCH 239/433] Factor out linkOrCopy() --- src/libstore/build.cc | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 68c2f2ce3..886c864a3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1874,6 +1874,21 @@ static void preloadNSS() { }); } + +void linkOrCopy(const Path & from, const Path & to) +{ + if (link(from.c_str(), to.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum link count on a + file (e.g. 32000 of ext3), which is quite possible after a + 'nix-store --optimise'. FIXME: actually, why don't we just + bind-mount in this case? */ + if (errno != EMLINK) + throw SysError("linking '%s' to '%s'", to, from); + copyPath(from, to); + } +} + + void DerivationGoal::startBuilder() { /* Right platform? */ @@ -2117,22 +2132,8 @@ void DerivationGoal::startBuilder() throw SysError(format("getting attributes of path '%1%'") % i); if (S_ISDIR(st.st_mode)) dirsInChroot[i] = r; - else { - Path p = chrootRootDir + i; - debug("linking '%1%' to '%2%'", p, r); - if (link(r.c_str(), p.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum - link count on a file (e.g. 32000 of ext3), - which is quite possible after a `nix-store - --optimise'. */ - if (errno != EMLINK) - throw SysError(format("linking '%1%' to '%2%'") % p % i); - StringSink sink; - dumpPath(r, sink); - StringSource source(*sink.s); - restorePath(p, source); - } - } + else + linkOrCopy(r, chrootRootDir + i); } /* If we're repairing, checking or rebuilding part of a From ec448f8bb694b6f9546e49fe6a79b86ff2b2f90a Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 28 Oct 2019 21:22:38 +0100 Subject: [PATCH 240/433] libexpr: findDerivationFilename return Pos instead of tuple --- src/libexpr/attr-path.cc | 8 +++++--- src/libexpr/attr-path.hh | 6 ++---- src/nix/edit.cc | 6 +++--- src/nix/repl.cc | 4 +++- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 7a6d8dfd0..c86dbeebb 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -93,7 +93,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, } -std::tuple findDerivationFilename(EvalState & state, Value & v, std::string what) +Pos findDerivationFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -110,14 +110,16 @@ std::tuple findDerivationFilename(EvalState & state, Value & v throw Error("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); - int lineno; + unsigned int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } - return std::make_tuple(filename, lineno); + Symbol file = state.symbols.create(filename); + + return { file, lineno, 0 }; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index dca94cc78..716e5ba27 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -4,15 +4,13 @@ #include #include -#include namespace nix { Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a derivation. */ -std::tuple findDerivationFilename(EvalState & state, - Value & v, std::string what); +/* Heuristic to find the filename and lineno or a nix value. */ +Pos findDerivationFilename(EvalState & state, Value & v, std::string what); } diff --git a/src/nix/edit.cc b/src/nix/edit.cc index a4aa40bed..d0607747d 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -36,9 +36,9 @@ struct CmdEdit : InstallableCommand auto v = installable->toValue(*state); - std::string filename; - int lineno; - std::tie(filename, lineno) = findDerivationFilename(*state, *v, installable->what()); + Pos pos = findDerivationFilename(*state, *v, installable->what()); + std::string filename(pos.file); + int lineno(pos.line); stopProgressBar(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 5cfb408f7..ed67c285f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -481,7 +481,9 @@ bool NixRepl::processLine(string line) lineno = 0; } else { // assume it's a derivation - std::tie(filename, lineno) = findDerivationFilename(state, v, arg); + Pos pos = findDerivationFilename(state, v, arg); + filename = pos.file; + lineno = pos.line; } // Open in EDITOR From 3774fe55fd6c96e80cc91e13fe0a231ce836ac47 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 28 Oct 2019 21:36:34 +0100 Subject: [PATCH 241/433] editorFor: take a pos object instead --- src/libutil/args.cc | 8 ++++---- src/libutil/args.hh | 3 ++- src/nix/edit.cc | 4 +--- src/nix/repl.cc | 13 +++++-------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 35ec3e4ab..b7baad375 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -178,16 +178,16 @@ Strings argvToStrings(int argc, char * * argv) return args; } -Strings editorFor(std::string filename, int lineno) +Strings editorFor(Pos pos) { auto editor = getEnv("EDITOR", "cat"); auto args = tokenizeString(editor); - if (lineno > 0 && ( + if (pos.line > 0 && ( editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos || editor.find("vim") != std::string::npos)) - args.push_back(fmt("+%d", lineno)); - args.push_back(filename); + args.push_back(fmt("+%d", pos.line)); + args.push_back(pos.file); return args; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 22702c2d8..1e29bd4fa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -5,6 +5,7 @@ #include #include "util.hh" +#include "nixexpr.hh" namespace nix { @@ -191,7 +192,7 @@ public: Strings argvToStrings(int argc, char * * argv); /* Helper function to generate args that invoke $EDITOR on filename:lineno */ -Strings editorFor(std::string filename, int lineno); +Strings editorFor(Pos pos); /* Helper function for rendering argument labels. */ std::string renderLabels(const Strings & labels); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index d0607747d..553765f13 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -37,12 +37,10 @@ struct CmdEdit : InstallableCommand auto v = installable->toValue(*state); Pos pos = findDerivationFilename(*state, *v, installable->what()); - std::string filename(pos.file); - int lineno(pos.line); stopProgressBar(); - auto args = editorFor(filename, lineno); + auto args = editorFor(pos); execvp(args.front().c_str(), stringsToCharPtrs(args).data()); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index ed67c285f..683a117f3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -472,22 +472,19 @@ bool NixRepl::processLine(string line) Value v; evalString(arg, v); - std::string filename; - int lineno = 0; + Pos pos; if (v.type == tPath || v.type == tString) { PathSet context; - filename = state.coerceToString(noPos, v, context); - lineno = 0; + auto filename = state.coerceToString(noPos, v, context); + pos.file = state.symbols.create(filename); } else { // assume it's a derivation - Pos pos = findDerivationFilename(state, v, arg); - filename = pos.file; - lineno = pos.line; + pos = findDerivationFilename(state, v, arg); } // Open in EDITOR - auto args = editorFor(filename, lineno); + auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); runProgram(editor, args); From d407f4d15f86aca585e0edebc5bc74ab8b1bebd1 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 28 Oct 2019 21:37:22 +0100 Subject: [PATCH 242/433] nix repl: also handle lambda edit --- src/nix/repl.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 683a117f3..35c7aec66 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -478,6 +478,8 @@ bool NixRepl::processLine(string line) PathSet context; auto filename = state.coerceToString(noPos, v, context); pos.file = state.symbols.create(filename); + } else if (v.type == tLambda) { + pos = v.lambda.fun->pos; } else { // assume it's a derivation pos = findDerivationFilename(state, v, arg); From 9a2505965667267f03a8385926f3b31a47732ed5 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 28 Oct 2019 21:40:02 +0100 Subject: [PATCH 243/433] findDerivationFilename: add FIXME --- src/libexpr/attr-path.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index c86dbeebb..06b472d8b 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -103,6 +103,8 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) throw Error("package '%s' has no source location information", what); } + // FIXME: is it possible to extract the Pos object instead of doing this + // toString + parsing? auto pos = state.forceString(*v2); auto colon = pos.rfind(':'); From 95c727caef110c03900a77133eb3e873fca7f019 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 11:24:54 +0200 Subject: [PATCH 244/433] Remove the check against concurrent builds in the same process --- src/libstore/build.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 886c864a3..dfae8024a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4138,9 +4138,6 @@ void SubstitutionGoal::handleEOF(int fd) ////////////////////////////////////////////////////////////////////// -static bool working = false; - - Worker::Worker(LocalStore & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -4148,8 +4145,6 @@ Worker::Worker(LocalStore & store) , store(store) { /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; nrLocalBuilds = 0; lastWokenUp = steady_time_point::min(); permanentFailure = false; @@ -4161,8 +4156,6 @@ Worker::Worker(LocalStore & store) Worker::~Worker() { - working = false; - /* Explicitly get rid of all strong pointers now. After this all goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in From 2d37e88319441958b7eed876042a106e79c7c85d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Sep 2018 13:53:44 +0200 Subject: [PATCH 245/433] Move most of the daemon implementation to libstore --- src/libstore/daemon.cc | 798 +++++++++++++++++++++++++++++++ src/libstore/daemon.hh | 12 + src/libstore/worker-protocol.hh | 3 + src/nix-daemon/nix-daemon.cc | 802 +------------------------------- 4 files changed, 821 insertions(+), 794 deletions(-) create mode 100644 src/libstore/daemon.cc create mode 100644 src/libstore/daemon.hh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc new file mode 100644 index 000000000..2e1d10bcc --- /dev/null +++ b/src/libstore/daemon.cc @@ -0,0 +1,798 @@ +#include "daemon.hh" +#include "monitor-fd.hh" +#include "worker-protocol.hh" +#include "store-api.hh" +#include "local-store.hh" +#include "finally.hh" +#include "affinity.hh" +#include "archive.hh" +#include "derivations.hh" +#include "args.hh" + +namespace nix::daemon { + +Sink & operator << (Sink & sink, const Logger::Fields & fields) +{ + sink << fields.size(); + for (auto & f : fields) { + sink << f.type; + if (f.type == Logger::Field::tInt) + sink << f.i; + else if (f.type == Logger::Field::tString) + sink << f.s; + else abort(); + } + return sink; +} + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +struct TunnelLogger : public Logger +{ + FdSink & to; + + struct State + { + bool canSendStderr = false; + std::vector pendingMsgs; + }; + + Sync state_; + + unsigned int clientVersion; + + TunnelLogger(FdSink & to, unsigned int clientVersion) + : to(to), clientVersion(clientVersion) { } + + void enqueueMsg(const std::string & s) + { + auto state(state_.lock()); + + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else + state->pendingMsgs.push_back(s); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() + { + auto state(state_.lock()); + state->canSendStderr = true; + + for (auto & msg : state->pendingMsgs) + to(msg); + + state->pendingMsgs.clear(); + + to.flush(); + } + + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + { + auto state(state_.lock()); + + state->canSendStderr = false; + + if (success) + to << STDERR_LAST; + else { + to << STDERR_ERROR << msg; + if (status != 0) to << status; + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) + log(lvl, s + "..."); + return; + } + + StringSink buf; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + enqueueMsg(*buf.s); + } + + void stopActivity(ActivityId act) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_STOP_ACTIVITY << act; + enqueueMsg(*buf.s); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_RESULT << act << type << fields; + enqueueMsg(*buf.s); + } +}; + +struct TunnelSink : Sink +{ + Sink & to; + TunnelSink(Sink & to) : to(to) { } + virtual void operator () (const unsigned char * data, size_t len) + { + to << STDERR_WRITE; + writeString(data, len, to); + } +}; + +struct TunnelSource : BufferedSource +{ + Source & from; + BufferedSink & to; + TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } + size_t readUnbuffered(unsigned char * data, size_t len) override + { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } +}; + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + bool regular; + string s; + + RetrieveRegularNARSink() : regular(true) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } +}; + +static void performOp(TunnelLogger * logger, ref store, + bool trusted, unsigned int clientVersion, + Source & from, BufferedSink & to, unsigned int op) +{ + switch (op) { + + case wopIsValidPath: { + /* 'readStorePath' could raise an error leading to the connection + being closed. To be able to recover from an invalid path error, + call 'startWork' early, and do 'assertStorePath' afterwards so + that the 'Error' exception handler doesn't close the + connection. */ + Path path = readString(from); + logger->startWork(); + store->assertStorePath(path); + bool result = store->isValidPath(path); + logger->stopWork(); + to << result; + break; + } + + case wopQueryValidPaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->queryValidPaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopHasSubstitutes: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths({path}); + logger->stopWork(); + to << (res.find(path) != res.end()); + break; + } + + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopQueryPathHash: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet paths; + if (op == wopQueryReferences) + paths = store->queryPathInfo(path)->references; + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else paths = store->queryDerivationOutputs(path); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryDerivationOutputNames: { + Path path = readStorePath(*store, from); + logger->startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + logger->stopWork(); + to << names; + break; + } + + case wopQueryDeriver: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto deriver = store->queryPathInfo(path)->deriver; + logger->stopWork(); + to << deriver; + break; + } + + case wopQueryPathFromHashPart: { + string hashPart = readString(from); + logger->startWork(); + Path path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << path; + break; + } + + case wopAddToStore: { + bool fixed, recursive; + std::string s, baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(s); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + auto store2 = store.dynamic_pointer_cast(); + if (!store2) throw Error("operation is only supported by LocalStore"); + + Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + logger->stopWork(); + + to << path; + break; + } + + case wopAddTextToStore: { + string suffix = readString(from); + string s = readString(from); + PathSet refs = readStorePaths(*store, from); + logger->startWork(); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << path; + break; + } + + case wopExportPath: { + Path path = readStorePath(*store, from); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; + } + + case wopImportPaths: { + logger->startWork(); + TunnelSource source(from, to); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + to << paths; + break; + } + + case wopBuildPaths: { + PathSet drvs = readStorePaths(*store, from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; + } + + case wopBuildDerivation: { + Path drvPath = readStorePath(*store, from); + BasicDerivation drv; + readDerivation(from, *store, drv); + BuildMode buildMode = (BuildMode) readInt(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; + } + + case wopEnsurePath: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddTempRoot: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopSyncWithGC: { + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; + } + + case wopFindRoots: { + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); + + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction) readInt(from); + options.pathsToDelete = readStorePaths(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; + } + + case wopSetOptions: { + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); + verbosity = (Verbosity) readInt(from); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + overrides.emplace(name, value); + } + } + + logger->startWork(); + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + + logger->stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + Path path = absPath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos({path}, infos); + logger->stopWork(); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + to << 0; + else { + to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; + } + break; + } + + case wopQuerySubstitutablePathInfos: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto & i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; + } + break; + } + + case wopQueryAllValidPaths: { + logger->startWork(); + PathSet paths = store->queryAllValidPaths(); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryPathInfo: { + Path path = readStorePath(*store, from); + std::shared_ptr info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) + to << 1; + to << info->deriver << info->narHash.to_string(Base16, false) << info->references + << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << info->ca; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; + } + + case wopOptimiseStore: + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; + + case wopVerifyStore: { + bool checkContents, repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) + throw Error("you are not privileged to repair paths"); + bool errors = store->verifyStore(checkContents, (RepairFlag) repair); + logger->stopWork(); + to << errors; + break; + } + + case wopAddSignatures: { + Path path = readStorePath(*store, from); + StringSet sigs = readStrings(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; + } + + case wopNarFromPath: { + auto path = readStorePath(*store, from); + logger->startWork(); + logger->stopWork(); + dumpPath(path, to); + break; + } + + case wopAddToStoreNar: { + bool repair, dontCheckSigs; + ValidPathInfo info; + info.path = readStorePath(*store, from); + from >> info.deriver; + if (!info.deriver.empty()) + store->assertStorePath(info.deriver); + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + if (!trusted) + info.ultimate = false; + + std::string saved; + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from, to); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; + } + + case wopQueryMissing: { + PathSet targets = readStorePaths(*store, from); + logger->startWork(); + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); + logger->stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; + } + + default: + throw Error(format("invalid operation %1%") % op); + } +} + +void processConnection( + FdSource & from, + FdSink & to, + bool trusted, + const std::string & userName, + uid_t userId) +{ + MonitorFdHup monitor(from.fd); + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); + + if (clientVersion < 0x10a) + throw Error("the Nix client version is too old"); + + auto tunnelLogger = new TunnelLogger(to, clientVersion); + auto prevLogger = nix::logger; + logger = tunnelLogger; + + unsigned int opCount = 0; + + Finally finally([&]() { + _isInterrupted = false; + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + }); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); + + readInt(from); // obsolete reserveSpace + + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); + + try { + + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + + /* Open the store. */ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); + + store->createUser(userName, userId); + + tunnelLogger->stopWork(); + to.flush(); + + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (Interrupted & e) { + break; + } catch (EndOfFile & e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) throw; + } catch (std::bad_alloc & e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; + + } catch (std::exception & e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } +} + +} diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh new file mode 100644 index 000000000..6d4015e34 --- /dev/null +++ b/src/libstore/daemon.hh @@ -0,0 +1,12 @@ +#include "serialise.hh" + +namespace nix::daemon { + +void processConnection( + FdSource & from, + FdSink & to, + bool trusted, + const std::string & userName, + uid_t userId); + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5ebdfaf13..a03edd10f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -62,6 +62,9 @@ typedef enum { #define STDERR_RESULT 0x52534c54 +class Store; +class Source; + Path readStorePath(Store & store, Source & from); template T readStorePaths(Store & store, Source & from); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index cd18489b0..799d2bacf 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -2,14 +2,12 @@ #include "local-store.hh" #include "util.hh" #include "serialise.hh" -#include "worker-protocol.hh" #include "archive.hh" -#include "affinity.hh" #include "globals.hh" -#include "monitor-fd.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" +#include "daemon.hh" #include @@ -32,6 +30,7 @@ #endif using namespace nix; +using namespace nix::daemon; #ifndef __linux__ #define SPLICE_F_MOVE 0 @@ -53,793 +52,6 @@ static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t } #endif -static FdSource from(STDIN_FILENO); -static FdSink to(STDOUT_FILENO); - - -Sink & operator << (Sink & sink, const Logger::Fields & fields) -{ - sink << fields.size(); - for (auto & f : fields) { - sink << f.type; - if (f.type == Logger::Field::tInt) - sink << f.i; - else if (f.type == Logger::Field::tString) - sink << f.s; - else abort(); - } - return sink; -} - - -/* Logger that forwards log messages to the client, *if* we're in a - state where the protocol allows it (i.e., when canSendStderr is - true). */ -struct TunnelLogger : public Logger -{ - struct State - { - bool canSendStderr = false; - std::vector pendingMsgs; - }; - - Sync state_; - - unsigned int clientVersion; - - TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { } - - void enqueueMsg(const std::string & s) - { - auto state(state_.lock()); - - if (state->canSendStderr) { - assert(state->pendingMsgs.empty()); - try { - to(s); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - state->canSendStderr = false; - throw; - } - } else - state->pendingMsgs.push_back(s); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - - StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); - enqueueMsg(*buf.s); - } - - /* startWork() means that we're starting an operation for which we - want to send out stderr to the client. */ - void startWork() - { - auto state(state_.lock()); - state->canSendStderr = true; - - for (auto & msg : state->pendingMsgs) - to(msg); - - state->pendingMsgs.clear(); - - to.flush(); - } - - /* stopWork() means that we're done; stop sending stderr to the - client. */ - void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) - { - auto state(state_.lock()); - - state->canSendStderr = false; - - if (success) - to << STDERR_LAST; - else { - to << STDERR_ERROR << msg; - if (status != 0) to << status; - } - } - - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) { - if (!s.empty()) - log(lvl, s + "..."); - return; - } - - StringSink buf; - buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; - enqueueMsg(*buf.s); - } - - void stopActivity(ActivityId act) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_STOP_ACTIVITY << act; - enqueueMsg(*buf.s); - } - - void result(ActivityId act, ResultType type, const Fields & fields) override - { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; - StringSink buf; - buf << STDERR_RESULT << act << type << fields; - enqueueMsg(*buf.s); - } -}; - - -struct TunnelSink : Sink -{ - Sink & to; - TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) - { - to << STDERR_WRITE; - writeString(data, len, to); - } -}; - - -struct TunnelSource : BufferedSource -{ - Source & from; - TunnelSource(Source & from) : from(from) { } -protected: - size_t readUnbuffered(unsigned char * data, size_t len) override - { - to << STDERR_READ << len; - to.flush(); - size_t n = readString(data, len, from); - if (n == 0) throw EndOfFile("unexpected end-of-file"); - return n; - } -}; - - -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular; - string s; - - RetrieveRegularNARSink() : regular(true) { } - - void createDirectory(const Path & path) - { - regular = false; - } - - void receiveContents(unsigned char * data, unsigned int len) - { - s.append((const char *) data, len); - } - - void createSymlink(const Path & path, const string & target) - { - regular = false; - } -}; - - -static void performOp(TunnelLogger * logger, ref store, - bool trusted, unsigned int clientVersion, - Source & from, Sink & to, unsigned int op) -{ - switch (op) { - - case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - logger->startWork(); - store->assertStorePath(path); - bool result = store->isValidPath(path); - logger->stopWork(); - to << result; - break; - } - - case wopQueryValidPaths: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - PathSet res = store->queryValidPaths(paths); - logger->stopWork(); - to << res; - break; - } - - case wopHasSubstitutes: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths({path}); - logger->stopWork(); - to << (res.find(path) != res.end()); - break; - } - - case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - PathSet res = store->querySubstitutablePaths(paths); - logger->stopWork(); - to << res; - break; - } - - case wopQueryPathHash: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto hash = store->queryPathInfo(path)->narHash; - logger->stopWork(); - to << hash.to_string(Base16, false); - break; - } - - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); - logger->startWork(); - PathSet paths; - if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; - else if (op == wopQueryReferrers) - store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) - paths = store->queryValidDerivers(path); - else paths = store->queryDerivationOutputs(path); - logger->stopWork(); - to << paths; - break; - } - - case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); - logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); - logger->stopWork(); - to << names; - break; - } - - case wopQueryDeriver: { - Path path = readStorePath(*store, from); - logger->startWork(); - auto deriver = store->queryPathInfo(path)->deriver; - logger->stopWork(); - to << deriver; - break; - } - - case wopQueryPathFromHashPart: { - string hashPart = readString(from); - logger->startWork(); - Path path = store->queryPathFromHashPart(hashPart); - logger->stopWork(); - to << path; - break; - } - - case wopAddToStore: { - bool fixed, recursive; - std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; - } - HashType hashAlgo = parseHashType(s); - - TeeSource savedNAR(from); - RetrieveRegularNARSink savedRegular; - - if (recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNAR); - } else - parseDump(savedRegular, from); - - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - auto store2 = store.dynamic_pointer_cast(); - if (!store2) throw Error("operation is only supported by LocalStore"); - - Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); - logger->stopWork(); - - to << path; - break; - } - - case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); - PathSet refs = readStorePaths(*store, from); - logger->startWork(); - Path path = store->addTextToStore(suffix, s, refs, NoRepair); - logger->stopWork(); - to << path; - break; - } - - case wopExportPath: { - Path path = readStorePath(*store, from); - readInt(from); // obsolete - logger->startWork(); - TunnelSink sink(to); - store->exportPath(path, sink); - logger->stopWork(); - to << 1; - break; - } - - case wopImportPaths: { - logger->startWork(); - TunnelSource source(from); - Paths paths = store->importPaths(source, nullptr, - trusted ? NoCheckSigs : CheckSigs); - logger->stopWork(); - to << paths; - break; - } - - case wopBuildPaths: { - PathSet drvs = readStorePaths(*store, from); - BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode) readInt(from); - - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ - if (mode == bmRepair && !trusted) - throw Error("repairing is not allowed because you are not in 'trusted-users'"); - } - logger->startWork(); - store->buildPaths(drvs, mode); - logger->stopWork(); - to << 1; - break; - } - - case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); - BasicDerivation drv; - readDerivation(from, *store, drv); - BuildMode buildMode = (BuildMode) readInt(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to build derivations"); - auto res = store->buildDerivation(drvPath, drv, buildMode); - logger->stopWork(); - to << res.status << res.errorMsg; - break; - } - - case wopEnsurePath: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->ensurePath(path); - logger->stopWork(); - to << 1; - break; - } - - case wopAddTempRoot: { - Path path = readStorePath(*store, from); - logger->startWork(); - store->addTempRoot(path); - logger->stopWork(); - to << 1; - break; - } - - case wopAddIndirectRoot: { - Path path = absPath(readString(from)); - logger->startWork(); - store->addIndirectRoot(path); - logger->stopWork(); - to << 1; - break; - } - - case wopSyncWithGC: { - logger->startWork(); - store->syncWithGC(); - logger->stopWork(); - to << 1; - break; - } - - case wopFindRoots: { - logger->startWork(); - Roots roots = store->findRoots(!trusted); - logger->stopWork(); - - size_t size = 0; - for (auto & i : roots) - size += i.second.size(); - - to << size; - - for (auto & [target, links] : roots) - for (auto & link : links) - to << link << target; - - break; - } - - case wopCollectGarbage: { - GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(*store, from); - from >> options.ignoreLiveness >> options.maxFreed; - // obsolete fields - readInt(from); - readInt(from); - readInt(from); - - GCResults results; - - logger->startWork(); - if (options.ignoreLiveness) - throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); - logger->stopWork(); - - to << results.paths << results.bytesFreed << 0 /* obsolete */; - - break; - } - - case wopSetOptions: { - settings.keepFailed = readInt(from); - settings.keepGoing = readInt(from); - settings.tryFallback = readInt(from); - verbosity = (Verbosity) readInt(from); - settings.maxBuildJobs.assign(readInt(from)); - settings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - settings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - settings.buildCores = readInt(from); - settings.useSubstitutes = readInt(from); - - StringMap overrides; - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); - for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); - overrides.emplace(name, value); - } - } - - logger->startWork(); - - for (auto & i : overrides) { - auto & name(i.first); - auto & value(i.second); - - auto setSubstituters = [&](Setting & res) { - if (name != res.name && res.aliases.count(name) == 0) - return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; - auto ss = tokenizeString(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else - warn("ignoring untrusted substituter '%s'", s); - res = subs; - return true; - }; - - try { - if (name == "ssh-auth-sock") // obsolete - ; - else if (trusted - || name == settings.buildTimeout.name - || name == "connect-timeout" - || (name == "builders" && value == "")) - settings.set(name, value); - else if (setSubstituters(settings.substituters)) - ; - else if (setSubstituters(settings.extraSubstituters)) - ; - else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); - } catch (UsageError & e) { - warn(e.what()); - } - } - - logger->stopWork(); - break; - } - - case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); - logger->stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); - if (i == infos.end()) - to << 0; - else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; - } - break; - } - - case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths(*store, from); - logger->startWork(); - SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(paths, infos); - logger->stopWork(); - to << infos.size(); - for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; - } - break; - } - - case wopQueryAllValidPaths: { - logger->startWork(); - PathSet paths = store->queryAllValidPaths(); - logger->stopWork(); - to << paths; - break; - } - - case wopQueryPathInfo: { - Path path = readStorePath(*store, from); - std::shared_ptr info; - logger->startWork(); - try { - info = store->queryPathInfo(path); - } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; - } - logger->stopWork(); - if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; - to << info->deriver << info->narHash.to_string(Base16, false) << info->references - << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << info->ca; - } - } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; - } - break; - } - - case wopOptimiseStore: - logger->startWork(); - store->optimiseStore(); - logger->stopWork(); - to << 1; - break; - - case wopVerifyStore: { - bool checkContents, repair; - from >> checkContents >> repair; - logger->startWork(); - if (repair && !trusted) - throw Error("you are not privileged to repair paths"); - bool errors = store->verifyStore(checkContents, (RepairFlag) repair); - logger->stopWork(); - to << errors; - break; - } - - case wopAddSignatures: { - Path path = readStorePath(*store, from); - StringSet sigs = readStrings(from); - logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); - store->addSignatures(path, sigs); - logger->stopWork(); - to << 1; - break; - } - - case wopNarFromPath: { - auto path = readStorePath(*store, from); - logger->startWork(); - logger->stopWork(); - dumpPath(path, to); - break; - } - - case wopAddToStoreNar: { - bool repair, dontCheckSigs; - ValidPathInfo info; - info.path = readStorePath(*store, from); - from >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); - info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - from >> info.ca >> repair >> dontCheckSigs; - if (!trusted && dontCheckSigs) - dontCheckSigs = false; - if (!trusted) - info.ultimate = false; - - std::string saved; - std::unique_ptr source; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from); - else { - TeeSink tee(from); - parseDump(tee, tee.source); - saved = std::move(*tee.source.data); - source = std::make_unique(saved); - } - - logger->startWork(); - - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, *source, (RepairFlag) repair, - dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); - - logger->stopWork(); - break; - } - - case wopQueryMissing: { - PathSet targets = readStorePaths(*store, from); - logger->startWork(); - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); - logger->stopWork(); - to << willBuild << willSubstitute << unknown << downloadSize << narSize; - break; - } - - default: - throw Error(format("invalid operation %1%") % op); - } -} - - -static void processConnection(bool trusted, - const std::string & userName, uid_t userId) -{ - MonitorFdHup monitor(from.fd); - - /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - unsigned int clientVersion = readInt(from); - - if (clientVersion < 0x10a) - throw Error("the Nix client version is too old"); - - auto tunnelLogger = new TunnelLogger(clientVersion); - auto prevLogger = nix::logger; - logger = tunnelLogger; - - unsigned int opCount = 0; - - Finally finally([&]() { - _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); - }); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); - - readInt(from); // obsolete reserveSpace - - /* Send startup error messages to the client. */ - tunnelLogger->startWork(); - - try { - - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - - /* Open the store. */ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - auto store = openStore(settings.storeUri, params); - - store->createUser(userName, userId); - - tunnelLogger->stopWork(); - to.flush(); - - /* Process client requests. */ - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (Interrupted & e) { - break; - } catch (EndOfFile & e) { - break; - } - - opCount++; - - try { - performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; - tunnelLogger->stopWork(false, e.msg(), e.status); - if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); - throw; - } - - to.flush(); - - assert(!tunnelLogger->state_.lock()->canSendStderr); - }; - - } catch (std::exception & e) { - tunnelLogger->stopWork(false, e.what(), 1); - to.flush(); - return; - } -} - static void sigChldHandler(int sigNo) { @@ -1054,9 +266,9 @@ static void daemonLoop(char * * argv) } /* Handle the connection. */ - from.fd = remote.get(); - to.fd = remote.get(); - processConnection(trusted, user, peer.uid); + FdSource from(remote.get()); + FdSink to(remote.get()); + processConnection(from, to, trusted, user, peer.uid); exit(0); }, options); @@ -1136,7 +348,9 @@ static int _main(int argc, char * * argv) } } } else { - processConnection(true, "root", 0); + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(from, to, true, "root", 0); } } else { daemonLoop(argv); From 63b99af85a1e280876da3d807e4bc96a5c5dde39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 12:36:11 +0200 Subject: [PATCH 246/433] Move Unix domain socket creation to libutil Also drop multithread-unfriendly hacks like doing a temporary chmod/umask. --- src/libutil/util.cc | 31 +++++++++++++++++++++++++ src/libutil/util.hh | 4 ++++ src/nix-daemon/nix-daemon.cc | 44 +++--------------------------------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2e416edef..2a1272885 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #ifdef __APPLE__ @@ -1562,4 +1564,33 @@ std::unique_ptr createInterruptCallback(std::function return std::unique_ptr(res.release()); } + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + + closeOnExec(fdSocket.get()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (path.size() >= sizeof(addr.sun_path)) + throw Error("socket path '%1%' is too long", path); + strcpy(addr.sun_path, path.c_str()); + + unlink(path.c_str()); + + if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%1%'", path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 5) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3493e80b5..0bb171268 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -553,4 +553,8 @@ typedef std::function PathFilter; extern PathFilter defaultPathFilter; +/* Create a Unix domain socket in listen mode. */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + + } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 799d2bacf..67bf4dfeb 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -156,53 +156,15 @@ static void daemonLoop(char * * argv) if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; + closeOnExec(fdSocket.get()); } /* Otherwise, create and bind to a Unix domain socket. */ else { - - /* Create and bind to a Unix domain socket. */ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - - string socketPath = settings.nixDaemonSocketFile; - - createDirs(dirOf(socketPath)); - - /* Urgh, sockaddr_un allows path names of only 108 characters. - So chdir to the socket directory so that we can pass a - relative path name. */ - if (chdir(dirOf(socketPath).c_str()) == -1) - throw SysError("cannot change current directory"); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); - - unlink(socketPath.c_str()); - - /* Make sure that the socket is created with 0666 permission - (everybody can connect --- provided they have access to the - directory containing the socket). */ - mode_t oldMode = umask(0111); - int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)); - umask(oldMode); - if (res == -1) - throw SysError(format("cannot bind to socket '%1%'") % socketPath); - - if (chdir("/") == -1) /* back to the root */ - throw SysError("cannot change current directory"); - - if (listen(fdSocket.get(), 5) == -1) - throw SysError(format("cannot listen on socket '%1%'") % socketPath); + createDirs(dirOf(settings.nixDaemonSocketFile)); + fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - closeOnExec(fdSocket.get()); - /* Loop accepting connections. */ while (1) { From 05819d013fb39eb9902d952978fff7f69e0e326e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 12:49:20 +0200 Subject: [PATCH 247/433] Don't create a Store in processConnection() --- src/libstore/daemon.cc | 7 +------ src/libstore/daemon.hh | 2 ++ src/nix-daemon/nix-daemon.cc | 13 +++++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2e1d10bcc..cc1436f90 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -694,6 +694,7 @@ static void performOp(TunnelLogger * logger, ref store, } void processConnection( + ref store, FdSource & from, FdSink & to, bool trusted, @@ -743,12 +744,6 @@ void processConnection( throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - /* Open the store. */ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - auto store = openStore(settings.storeUri, params); - store->createUser(userName, userId); tunnelLogger->stopWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 6d4015e34..fa26ba9c2 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,8 +1,10 @@ #include "serialise.hh" +#include "store-api.hh" namespace nix::daemon { void processConnection( + ref store, FdSource & from, FdSink & to, bool trusted, diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 67bf4dfeb..da17ddcba 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -140,6 +140,15 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +static ref openUncachedStore() +{ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + return openStore(settings.storeUri, params); +} + + static void daemonLoop(char * * argv) { if (chdir("/") == -1) @@ -230,7 +239,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(from, to, trusted, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, user, peer.uid); exit(0); }, options); @@ -312,7 +321,7 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(from, to, true, "root", 0); + processConnection(openUncachedStore(), from, to, true, "root", 0); } } else { daemonLoop(argv); From 992a2ad4757f43c02f3d06771c632a408d90ded0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 18:55:37 +0200 Subject: [PATCH 248/433] Move addToStoreFromDump to Store --- src/libstore/daemon.cc | 5 +---- src/libstore/local-store.hh | 2 +- src/libstore/store-api.hh | 7 +++++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cc1436f90..64147027b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -316,10 +316,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto store2 = store.dynamic_pointer_cast(); - if (!store2) throw Error("operation is only supported by LocalStore"); - - Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); logger->stopWork(); to << path; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 379a06af8..77253fa4a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -157,7 +157,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ Path addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; Path addTextToStore(const string & name, const string & s, const PathSet & references, RepairFlag repair) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 250e1a4bc..115b23cbf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -422,6 +422,13 @@ public: bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + // FIXME: remove? + virtual Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + { + throw Error("addToStoreFromDump() is not supported by this store"); + } + /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ virtual Path addTextToStore(const string & name, const string & s, From e5319a87ce75bbd2dd88f57c3b470a396195e849 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Sep 2018 18:54:16 +0200 Subject: [PATCH 249/433] queryPathInfoUncached(): Return const ValidPathInfo --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/binary-cache-store.hh | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info-disk-cache.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 16 ++++++++-------- src/libstore/store-api.hh | 6 +++--- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8e6f1f55d..7e3e5ff88 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -296,7 +296,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback> callback) noexcept + Callback> callback) noexcept { auto uri = getUri(); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c77292294..2d7cd1947 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -74,7 +74,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d5fbdd25a..1b8b5908c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override + Callback> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 49061f27d..6bbe5433c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -632,7 +632,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback> callback) noexcept + Callback> callback) noexcept { try { auto info = std::make_shared(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 77253fa4a..5aa6b0519 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 05b93d4c9..2fceb9b9a 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,7 +33,7 @@ void Store::computeFSClosure(const PathSet & startPaths, state->pending++; } - queryPathInfo(path, {[&, path](std::future> fut) { + queryPathInfo(path, {[&, path](std::future> fut) { // FIXME: calls to isValidPath() should be async try { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 32ad7f2b2..5bf982195 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -219,7 +219,7 @@ public: void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) override + std::shared_ptr info) override { retrySQLite([&]() { auto state(_state.lock()); @@ -228,7 +228,7 @@ public: if (info) { - auto narInfo = std::dynamic_pointer_cast(info); + auto narInfo = std::dynamic_pointer_cast(info); assert(hashPart == storePathToHash(info->path)); diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 88d909732..11e6c55ca 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -21,7 +21,7 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, - std::shared_ptr info) = 0; + std::shared_ptr info) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f34369d8f..2a89b7c98 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -350,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback> callback) noexcept + Callback> callback) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1f375dd71..84de4c460 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -43,7 +43,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a1b24e3b..54430d3ba 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -320,10 +320,10 @@ bool Store::isValidPathUncached(const Path & path) ref Store::queryPathInfo(const Path & storePath) { - std::promise> promise; + std::promise> promise; queryPathInfo(storePath, - {[&](std::future> result) { + {[&](std::future> result) { try { promise.set_value(result.get()); } catch (...) { @@ -336,7 +336,7 @@ ref Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback> callback) noexcept + Callback> callback) noexcept { std::string hashPart; @@ -351,7 +351,7 @@ void Store::queryPathInfo(const Path & storePath, stats.narInfoReadAverted++; if (!*res) throw InvalidPath(format("path '%s' is not valid") % storePath); - return callback(ref(*res)); + return callback(ref(*res)); } } @@ -367,7 +367,7 @@ void Store::queryPathInfo(const Path & storePath, (res.second->path != storePath && storePathToName(storePath) != "")) throw InvalidPath(format("path '%s' is not valid") % storePath); } - return callback(ref(res.second)); + return callback(ref(res.second)); } } @@ -376,7 +376,7 @@ void Store::queryPathInfo(const Path & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -396,7 +396,7 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - (*callbackPtr)(ref(info)); + (*callbackPtr)(ref(info)); } catch (...) { callbackPtr->rethrow(); } }}); } @@ -418,7 +418,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto doQuery = [&](const Path & path ) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 115b23cbf..e9e6e0dd2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -258,7 +258,7 @@ protected: struct State { - LRUCache> pathInfoCache; + LRUCache> pathInfoCache; }; Sync state; @@ -361,12 +361,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback> callback) noexcept; + Callback> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback> callback) noexcept = 0; + Callback> callback) noexcept = 0; public: From 0e9b72e097e027ff7f78e0f596a3ade606df9253 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 30 Oct 2019 14:38:02 -0500 Subject: [PATCH 250/433] minor: fix mismatch of struct/class forward decl of 'Source' Fixes the following warning and the indicate potential issue: src/libstore/worker-protocol.hh:66:1: warning: class 'Source' was previously declared as a struct; this is valid, but may result in linker errors under the Microsoft C++ ABI [-Wmismatched-tags] (cherry picked from commit 6e1bb04870b1b723282d32182af286646f13bf3c) --- src/libstore/worker-protocol.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index a03edd10f..6762b609d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -63,7 +63,7 @@ typedef enum { class Store; -class Source; +struct Source; Path readStorePath(Store & store, Source & from); template T readStorePaths(Store & store, Source & from); From 99aac72a16544637e9ab2e14c7b28f64b4638c8e Mon Sep 17 00:00:00 2001 From: Kevin Stock Date: Wed, 30 Oct 2019 16:53:04 -0400 Subject: [PATCH 251/433] docs: fix upper bound on number of consumed cores --- doc/manual/advanced-topics/cores-vs-jobs.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml index eba645faf..4d58ac7c5 100644 --- a/doc/manual/advanced-topics/cores-vs-jobs.xml +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -36,8 +36,8 @@ to , unless equals 0, in which case NIX_BUILD_CORES will be the total number of cores in the system. -The total number of consumed cores is a simple multiplication, - * NIX_BUILD_CORES. +The maximum number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. The balance on how to set these two independent variables depends upon each builder's workload and hardware. Here are a few example From 80d5ec6ff4388b258278ee507b94694dc34d19f5 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sun, 27 Oct 2019 13:45:02 -0400 Subject: [PATCH 252/433] Minor updates to inline comments Add missing docstring on InstallableCommand. Also, some of these were wrapped when they're right next to a line longer than the unwrapped line, so we can just unwrap them to save vertical space. --- src/libexpr/primops/fetchGit.cc | 4 +--- src/libstore/build.cc | 12 ++++-------- src/libstore/local-store.cc | 3 +-- src/nix-store/nix-store.cc | 6 ++---- src/nix/command.hh | 3 ++- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 90f600284..7ef3b3823 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -45,9 +45,7 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { - /* This is an unclean working tree. So copy all tracked - files. */ - + /* This is an unclean working tree. So copy all tracked files. */ GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dfae8024a..ada1cf8ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3264,8 +3264,7 @@ void DerivationGoal::registerOutputs() i.second.parseHashInfo(recursive, h); if (!recursive) { - /* The output path should be a regular file without - execute permission. */ + /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( format("output path '%1%' should be a non-executable regular file") % path); @@ -3343,8 +3342,7 @@ void DerivationGoal::registerOutputs() % drvPath % path); } - /* Since we verified the build, it's now ultimately - trusted. */ + /* Since we verified the build, it's now ultimately trusted. */ if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); @@ -3354,8 +3352,7 @@ void DerivationGoal::registerOutputs() continue; } - /* For debugging, print out the referenced and unreferenced - paths. */ + /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { PathSet::iterator j = references.find(i); if (j == references.end()) @@ -3413,8 +3410,7 @@ void DerivationGoal::registerOutputs() } } - /* If this is the first round of several, then move the output out - of the way. */ + /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6bbe5433c..b5b0f1ba7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -937,8 +937,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) not be valid yet. */ for (auto & i : infos) if (isDerivation(i.path)) { - // FIXME: inefficient; we already loaded the - // derivation in addValidPath(). + // FIXME: inefficient; we already loaded the derivation in addValidPath(). Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); checkDerivationOutputs(i.path, drv); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 19384499d..dfa473db5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -632,8 +632,7 @@ static void opDelete(Strings opFlags, Strings opArgs) } -/* Dump a path as a Nix archive. The archive is written to standard - output. */ +/* Dump a path as a Nix archive. The archive is written to stdout */ static void opDump(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -646,8 +645,7 @@ static void opDump(Strings opFlags, Strings opArgs) } -/* Restore a value from a Nix archive. The archive is read from - standard input. */ +/* Restore a value from a Nix archive. The archive is read from stdin. */ static void opRestore(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); diff --git a/src/nix/command.hh b/src/nix/command.hh index 45ad1cd2a..fad404c73 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -34,7 +34,7 @@ struct Command : virtual Args class Store; -/* A command that require a Nix store. */ +/* A command that requires a Nix store. */ struct StoreCommand : virtual Command { StoreCommand(); @@ -115,6 +115,7 @@ private: std::vector _installables; }; +/* A command that operates on exactly one "installable" */ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr installable; From f1782642d363330b9ce8607d942dcbb47414bf82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 31 Oct 2019 16:34:59 +0000 Subject: [PATCH 253/433] travis: enable linux builds Also disable email to not notify the whole NixOS community about build failures --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99218a963..ee4ea1ac6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,8 @@ -os: osx -script: ./tests/install-darwin.sh +matrix: + include: + - language: osx + script: ./tests/install-darwin.sh + - language: nix + script: nix-build release.nix -A build.x86_64-linux +notifications: + email: false From b811bd2172bc9796f42b87996f455d4e5e4382ba Mon Sep 17 00:00:00 2001 From: ng0 Date: Fri, 1 Nov 2019 14:09:42 +0000 Subject: [PATCH 254/433] include netinet/in.h in src/nix/main.cc Fixes #3186 --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 22f5e8c4c..1c9d909d8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -15,6 +15,7 @@ #include #include #include +#include extern std::string chrootHelperName; From f1d4ba2afd1169b6c15248e50e30b592e514bb21 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 1 Nov 2019 13:25:15 -0400 Subject: [PATCH 255/433] Update man to show that nix-shell allows --arg --- doc/manual/command-ref/opt-common.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index b8a2f260e..28ec42bbc 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -243,9 +243,10 @@ name value This option is accepted by - nix-env, nix-instantiate and - nix-build. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that + nix-env, nix-instantiate, + nix-shellnix-build. + When evaluating Nix expressions, the expression evaluator will + automatically try to call functions that it encounters. It can automatically call functions for which every argument has a default value (e.g., { argName ? From 3e85c57a6cbf46d5f0fe8a89b368a43abd26daba Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 1 Nov 2019 13:27:40 -0400 Subject: [PATCH 256/433] Pass --static flag to pkg-config when necessary --- configure.ac | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index ebe6d4267..20969c14f 100644 --- a/configure.ac +++ b/configure.ac @@ -170,6 +170,18 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi +PKG_PROG_PKG_CONFIG + +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) + PKG_CONFIG="$PKG_CONFIG --static" +fi + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -302,16 +314,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), - shared=$enableval, shared=yes) -if test "$shared" = yes; then - AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) -else - AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) -fi - - # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' From 808cb6444e33c413fad220d2cf9c565fa68614f0 Mon Sep 17 00:00:00 2001 From: Kevin Stock Date: Sat, 2 Nov 2019 17:55:53 -0400 Subject: [PATCH 257/433] docs: xref doesn't render in title The `post-build-hook` text currently appears in the index, but not on the actual title line of the section, this follows the pattern used in a previous section to get a reference into a title. --- doc/manual/advanced-topics/post-build-hook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 3dc43ee79..08a7a772f 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -5,7 +5,7 @@ version="5.0" > -Using the <xref linkend="conf-post-build-hook" /> +Using the <option linkend="conf-post-build-hook">post-build-hook</option> Uploading to an S3-compatible binary cache after each build From dcd7a260639c5130368e5969a34aa9bb2365df17 Mon Sep 17 00:00:00 2001 From: kraem Date: Sun, 3 Nov 2019 18:47:28 +0100 Subject: [PATCH 258/433] Update nix eval --help msg to not include deprecated command --- src/nix/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b7058361c..daefea757 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -41,7 +41,7 @@ struct CmdEval : MixJSON, InstallableCommand }, Example{ "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.nixpkgsVersion" + "nix eval --raw nixpkgs.lib.version" }, Example{ "To print the store path of the Hello package:", From c935ad3f025d5c3d8026711a1eb50b2917b61d59 Mon Sep 17 00:00:00 2001 From: Harald van Dijk Date: Sun, 3 Nov 2019 21:46:59 +0000 Subject: [PATCH 259/433] Fix progress bar when nix-prefetch-url is piped. The intent of the code was that if the window size cannot be determined, it would be treated as having the maximum possible size. Because of a missing assignment, it was actually treated as having a width of 0. The reason the width could not be determined was because it was obtained from stdout, not stderr, even though the printing was done to stderr. This commit addresses both issues. --- src/libutil/util.cc | 2 +- src/nix/progress-bar.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2a1272885..d29802bcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1467,7 +1467,7 @@ static Sync> windowSize{{0, 0}}; static void updateWindowSize() { struct winsize ws; - if (ioctl(1, TIOCGWINSZ, &ws) == 0) { + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { auto windowSize_(windowSize.lock()); windowSize_->first = ws.ws_row; windowSize_->second = ws.ws_col; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 5c05d6b22..661966733 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -341,7 +341,7 @@ public: } auto width = getWindowSize().second; - if (width <= 0) std::numeric_limits::max(); + if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); } From cea05e5ee758daad40047db0b861980d80da2e85 Mon Sep 17 00:00:00 2001 From: Kevin Stock Date: Mon, 4 Nov 2019 16:23:03 -0500 Subject: [PATCH 260/433] docs: correct default location of log directory --- doc/manual/command-ref/env-common.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 6a3aaae71..696d68c34 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -122,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix NIX_LOG_DIR Overrides the location of the Nix log directory - (default prefix/log/nix). + (default prefix/var/log/nix). From 1b600ecd14049985dc1001d22d6495810ca418a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Nov 2019 10:25:09 +0100 Subject: [PATCH 261/433] Don't use SOCK_CLOEXEC on macOS https://hydra.nixos.org/build/105428308 --- src/libutil/util.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d29802bcd..34c1d96dc 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1567,7 +1567,11 @@ std::unique_ptr createInterruptCallback(std::function AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) { - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); if (!fdSocket) throw SysError("cannot create Unix domain socket"); From e1725ba946aa89271bd8ef83f9ecc2e2dc6eb5df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Nov 2019 11:12:25 +0100 Subject: [PATCH 262/433] Fix VM tests --- tests/nix-copy-closure.nix | 4 ++-- tests/remote-builds.nix | 4 ++-- tests/setuid.nix | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 0dc147fb3..bb5db7410 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -8,11 +8,11 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = - { config, pkgs, ... }: + { config, lib, pkgs, ... }: { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgA ]; nix.package = nix; - nix.binaryCaches = [ ]; + nix.binaryCaches = lib.mkForce [ ]; }; server = diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index b867f13b4..18d490830 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -40,7 +40,7 @@ in builder2 = builder; client = - { config, pkgs, ... }: + { config, lib, pkgs, ... }: { nix.maxJobs = 0; # force remote building nix.distributedBuilds = true; nix.buildMachines = @@ -60,7 +60,7 @@ in virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; nix.package = nix; - nix.binaryCaches = [ ]; + nix.binaryCaches = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; }; }; diff --git a/tests/setuid.nix b/tests/setuid.nix index 77e83c8d6..63d3c05cb 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -10,7 +10,7 @@ makeTest { { config, lib, pkgs, ... }: { virtualisation.writableStore = true; nix.package = nix; - nix.binaryCaches = [ ]; + nix.binaryCaches = lib.mkForce [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; }; From 81a9b93689c76a9b90134bcc5ad8a6b851590599 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Nov 2019 11:21:32 +0100 Subject: [PATCH 263/433] Fix manual build --- doc/manual/command-ref/opt-common.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 28ec42bbc..6a8107be9 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -244,7 +244,7 @@ This option is accepted by nix-env, nix-instantiate, - nix-shellnix-build. + nix-shell and nix-build. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every From b4e260d887441fde9ab568dff7c21a77d7cff904 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Nov 2019 16:00:30 +0100 Subject: [PATCH 264/433] Disable shellcheck It's broken at the moment: https://hydra.nixos.org/build/105746055 Also it pulls in GHC which is a pretty big dependency. --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 8fcf04680..e512a7671 100644 --- a/release.nix +++ b/release.nix @@ -128,7 +128,7 @@ let in runCommand "nix-binary-tarball-${version}" - { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' From b874272f7a121a859e7c05abe4d7818bc852eba8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Oct 2019 14:43:09 +0100 Subject: [PATCH 265/433] Make --enable-gc the default --- configure.ac | 4 ++-- release-common.nix | 4 +--- release.nix | 2 -- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index ebe6d4267..4a6eaac8a 100644 --- a/configure.ac +++ b/configure.ac @@ -255,8 +255,8 @@ fi # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], - [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]), - gc=$enableval, gc=no) + [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), + gc=$enableval, gc=yes) if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" diff --git a/release-common.nix b/release-common.nix index 2e8a951b9..11944a631 100644 --- a/release-common.nix +++ b/release-common.nix @@ -30,9 +30,7 @@ rec { }); configureFlags = - [ - "--enable-gc" - ] ++ lib.optionals stdenv.isLinux [ + lib.optionals stdenv.isLinux [ "--with-sandbox-shell=${sh}/bin/busybox" ]; diff --git a/release.nix b/release.nix index e512a7671..0b74de390 100644 --- a/release.nix +++ b/release.nix @@ -25,8 +25,6 @@ let buildInputs = tarballDeps ++ buildDeps; - configureFlags = "--enable-gc"; - postUnpack = '' (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files cat $sourceRoot/.dist-files From c4d7c76b641d82b2696fef73ce0ac160043c18da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 16:01:26 +0200 Subject: [PATCH 266/433] Recursive Nix support This allows Nix builders to call Nix to build derivations, with some limitations. Example: let nixpkgs = fetchTarball channel:nixos-18.03; in with import {}; runCommand "foo" { buildInputs = [ nix jq ]; NIX_PATH = "nixpkgs=${nixpkgs}"; } '' hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "hello-3.5"; })') $hello/bin/hello mkdir -p $out/bin ln -s $hello/bin/hello $out/bin/hello nix path-info -r --json $hello | jq . '' This derivation makes a recursive Nix call to build GNU Hello and symlinks it from its $out, i.e. # ll ./result/bin/ lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello # nix-store -qR ./result /nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131 /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5 /nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo This is implemented as follows: * Before running the outer builder, Nix creates a Unix domain socket '.nix-socket' in the builder's temporary directory and sets $NIX_REMOTE to point to it. It starts a thread to process connections to this socket. (Thus you don't need to have nix-daemon running.) * The daemon thread uses a wrapper store (RestrictedStore) to keep track of paths added through recursive Nix calls, to implement some restrictions (see below), and to do some censorship (e.g. for purity, queryPathInfo() won't return impure information such as signatures and timestamps). * After the build finishes, the output paths are scanned for references to the paths added through recursive Nix calls (in addition to the inputs closure). Thus, in the example above, $out has a reference to $hello. The main restriction on recursive Nix calls is that they cannot do arbitrary substitutions. For example, doing nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 is forbidden unless /nix/store/kmwd... is in the inputs closure or previously built by a recursive Nix call. This is to prevent irreproducible derivations that have hidden dependencies on substituters or the current store contents. Building a derivation is fine, however, and Nix will use substitutes if available. In other words, the builder has to present proof that it knows how to build a desired store path from scratch by constructing a derivation graph for that path. Probably we should also disallow instantiating/building fixed-output derivations (specifically, those that access the network, but currently we have no way to mark fixed-output derivations that don't access the network). Otherwise sandboxed derivations can bypass sandbox restrictions and access the network. When sandboxing is enabled, we make paths appear in the sandbox of the builder by entering the mount namespace of the builder and bind-mounting each path. This is tricky because we do a pivot_root() in the builder to change the root directory of its mount namespace, and thus the host /nix/store is not visible in the mount namespace of the builder. To get around this, just before doing pivot_root(), we branch a second mount namespace that shares its /nix/store mountpoint with the parent. Recursive Nix currently doesn't work on macOS in sandboxed mode (because we can't change the sandbox policy of a running build) and in non-root mode (because setns() barfs). --- src/libstore/build.cc | 403 +++++++++++++++++++++++++++++++++-- src/libstore/daemon.cc | 153 ++++++++----- src/libstore/daemon.hh | 6 +- src/nix-daemon/nix-daemon.cc | 8 +- 4 files changed, 492 insertions(+), 78 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ada1cf8ec..2fd27cfa0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -13,6 +13,7 @@ #include "nar-info.hh" #include "parsed-derivations.hh" #include "machines.hh" +#include "daemon.hh" #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -765,9 +767,6 @@ private: immediate input paths). */ PathSet inputPaths; - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; - /* Outputs that are already valid. If we're repairing, these are the outputs that are valid *and* not corrupt. */ PathSet validPaths; @@ -805,9 +804,13 @@ private: /* Pipe for the builder's standard output/error. */ Pipe builderOut; - /* Pipe for synchronising updates to the builder user namespace. */ + /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; + /* The mount namespace of the builder, used to add additional + paths to the sandbox as a result of recursive Nix calls. */ + AutoCloseFD sandboxMountNamespace; + /* The build hook. */ std::unique_ptr hook; @@ -886,6 +889,26 @@ private: /* The remote machine on which we're building. */ std::string machineName; + /* The recursive Nix daemon socket. */ + AutoCloseFD daemonSocket; + + /* The daemon main thread. */ + std::thread daemonThread; + + /* Paths that were added via recursive Nix calls. */ + PathSet addedPaths; + + /* Recursive Nix calls are only allowed to build or realize paths + in the original input closure or added via a recursive Nix call + (so e.g. you can't do 'nix-store -r /nix/store/' where + /nix/store/ is some arbitrary path in a binary cache). */ + bool isAllowed(const Path & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + + friend class RestrictedStore; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -942,6 +965,14 @@ private: /* Write a JSON file containing the derivation attributes. */ void writeStructuredAttrs(); + void startDaemon(); + + void stopDaemon(); + + /* Add 'path' to the set of paths that may be referenced by the + outputs, and make it appear in the sandbox. */ + void addDependency(const Path & path); + /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); @@ -1043,6 +1074,7 @@ DerivationGoal::~DerivationGoal() /* Careful: we should never ever throw an exception from a destructor. */ try { killChild(); } catch (...) { ignoreException(); } + try { stopDaemon(); } catch (...) { ignoreException(); } try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { closeLogFile(); } catch (...) { ignoreException(); } } @@ -1332,12 +1364,6 @@ void DerivationGoal::inputsRealised() /* Gather information necessary for computing the closure and/or running the build hook. */ - /* The outputs are referenceable paths. */ - for (auto & i : drv->outputs) { - debug(format("building path '%1%'") % i.second.path); - allPaths.insert(i.second.path); - } - /* Determine the full set of input paths. */ /* First, the input derivations. */ @@ -1362,8 +1388,6 @@ void DerivationGoal::inputsRealised() debug(format("added input paths %1%") % showPaths(inputPaths)); - allPaths.insert(inputPaths.begin(), inputPaths.end()); - /* Is this a fixed-output derivation? */ fixedOutput = drv->isFixedOutput(); @@ -1527,6 +1551,8 @@ void DerivationGoal::buildDone() uid and then messing around with our output. */ Finally releaseBuildUser([&]() { buildUser.reset(); }); + sandboxMountNamespace = -1; + /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, @@ -1558,6 +1584,9 @@ void DerivationGoal::buildDone() root. */ if (buildUser) buildUser->kill(); + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + bool diskFull = false; try { @@ -2217,6 +2246,10 @@ void DerivationGoal::startBuilder() } } + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + startDaemon(); + /* Run the builder. */ printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); @@ -2391,6 +2424,12 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); + /* Save the mount namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; @@ -2621,6 +2660,302 @@ void DerivationGoal::writeStructuredAttrs() } +/* A wrapper around LocalStore that only allows building/querying of + paths that are in the input closures of the build or were added via + recursive Nix calls. */ +struct RestrictedStore : public LocalFSStore +{ + ref next; + + DerivationGoal & goal; + + RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + : Store(params), LocalFSStore(params), next(next), goal(goal) + { } + + Path getRealStoreDir() override + { return next->realStoreDir; } + + std::string getUri() override + { return next->getUri(); } + + PathSet queryAllValidPaths() override + { + PathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); + return paths; + } + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override + { + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.clear(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); + }; + + void queryReferrers(const Path & path, PathSet & referrers) override + { } + + PathSet queryDerivationOutputs(const Path & path) override + { throw Error("queryDerivationOutputs"); } + + StringSet queryDerivationOutputNames(const Path & path) override + { throw Error("queryDerivationOutputNames"); } + + Path queryPathFromHashPart(const string & hashPart) override + { throw Error("queryPathFromHashPart"); } + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + { throw Error("addToStore"); + } + + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + goal.addDependency(path); + return path; + } + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair = NoRepair) override + { + auto path = next->addTextToStore(name, s, references, repair); + goal.addDependency(path); + return path; + } + + void narFromPath(const Path & path, Sink & sink) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path); + LocalFSStore::narFromPath(path, sink); + } + + void ensurePath(const Path & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path); + /* Nothing to be done; 'path' must already be valid. */ + } + + void buildPaths(const PathSet & paths, BuildMode buildMode) override + { + if (buildMode != bmNormal) throw Error("unsupported build mode"); + + PathSet newPaths; + + for (auto & path : paths) { + DrvPathWithOutputs i = parseDrvPathWithOutputs(path); + if (isDerivation(i.first)) { + if (!goal.isAllowed(i.first)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first); + auto drv = derivationFromPath(i.first); + for (auto & output : drv.outputs) + if (wantOutput(output.first, i.second)) + newPaths.insert(output.second.path); + } else if (!goal.isAllowed(path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path); + } + + next->buildPaths(paths, buildMode); + + PathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + } + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + void addTempRoot(const Path & path) + { } + + void addIndirectRoot(const Path & path) + { } + + Roots findRoots() + { return Roots(); } + + void collectGarbage(const GCOptions & options, GCResults & results) + { } + + void addSignatures(const Path & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } + + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) + { + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + PathSet allowed; + for (auto & path : targets) { + DrvPathWithOutputs i = parseDrvPathWithOutputs(path); + if (goal.isAllowed(i.first)) + allowed.insert(i.first); + else + unknown.insert(i.first); + } + + next->queryMissing(allowed, willBuild, willSubstitute, + unknown, downloadSize, narSize); + } +}; + + +void DerivationGoal::startDaemon() +{ + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = worker.store.storeDir; + params["root"] = worker.store.rootDir; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = make_ref(params, + ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR) continue; + if (errno == EINVAL) break; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + debug("received daemon connection"); + + // FIXME: process on a separate thread. + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + } catch (SysError &) { + ignoreException(); + } + + debug("terminated daemon connection"); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationGoal::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) + throw SysError("shutting down daemon socket"); + + if (daemonThread.joinable()) + daemonThread.join(); + + daemonSocket = -1; +} + + +void DerivationGoal::addDependency(const Path & path) +{ + worker.store.assertStorePath(path); + + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", path); + + #if __linux__ + + Path source = worker.store.toRealPath(path); + Path target = chrootRootDir + path; + debug("bind-mounting %s -> %s", target, source); + + if (pathExists(target)) + throw Error("store path '%s' already exists in the sandbox", path); + + struct stat st; + if (lstat(source.c_str(), &st)) + throw SysError("getting attributes of path '%s'", source); + + if (S_ISDIR(st.st_mode)) { + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + createDirs(target); + + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", path); + + } else + linkOrCopy(source, target); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path); + #endif + + } +} + + void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; @@ -2756,15 +3091,30 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { - throw SysError("unable to make '/' private mount"); - } + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; @@ -2866,6 +3216,19 @@ void DerivationGoal::runChild() } } + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); @@ -3182,6 +3545,14 @@ void DerivationGoal::registerOutputs() std::exception_ptr delayedException; + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + PathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -3317,7 +3688,7 @@ void DerivationGoal::registerOutputs() verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); HashResult hash; - PathSet references = scanForReferences(actualPath, allPaths, hash); + PathSet references = scanForReferences(actualPath, referenceablePaths, hash); if (buildMode == bmCheck) { if (!worker.store.isValidPath(path)) continue; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 64147027b..d3c17e772 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -186,8 +186,75 @@ struct RetrieveRegularNARSink : ParseSink } }; +struct ClientSettings +{ + bool keepFailed; + bool keepGoing; + bool tryFallback; + Verbosity verbosity; + unsigned int maxBuildJobs; + time_t maxSilentTime; + bool verboseBuild; + unsigned int buildCores; + bool useSubstitutes; + StringMap overrides; + + void apply(TrustedFlag trusted) + { + settings.keepFailed = keepFailed; + settings.keepGoing = keepGoing; + settings.tryFallback = tryFallback; + nix::verbosity = verbosity; + settings.maxBuildJobs.assign(maxBuildJobs); + settings.maxSilentTime = maxSilentTime; + settings.verboseBuild = verboseBuild; + settings.buildCores = buildCores; + settings.useSubstitutes = useSubstitutes; + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + } +}; + static void performOp(TunnelLogger * logger, ref store, - bool trusted, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) { switch (op) { @@ -464,70 +531,37 @@ static void performOp(TunnelLogger * logger, ref store, } case wopSetOptions: { - settings.keepFailed = readInt(from); - settings.keepGoing = readInt(from); - settings.tryFallback = readInt(from); - verbosity = (Verbosity) readInt(from); - settings.maxBuildJobs.assign(readInt(from)); - settings.maxSilentTime = readInt(from); + + ClientSettings clientSettings; + + clientSettings.keepFailed = readInt(from); + clientSettings.keepGoing = readInt(from); + clientSettings.tryFallback = readInt(from); + clientSettings.verbosity = (Verbosity) readInt(from); + clientSettings.maxBuildJobs = readInt(from); + clientSettings.maxSilentTime = readInt(from); readInt(from); // obsolete useBuildHook - settings.verboseBuild = lvlError == (Verbosity) readInt(from); + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace - settings.buildCores = readInt(from); - settings.useSubstitutes = readInt(from); + clientSettings.buildCores = readInt(from); + clientSettings.useSubstitutes = readInt(from); - StringMap overrides; if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { unsigned int n = readInt(from); for (unsigned int i = 0; i < n; i++) { string name = readString(from); string value = readString(from); - overrides.emplace(name, value); + clientSettings.overrides.emplace(name, value); } } logger->startWork(); - for (auto & i : overrides) { - auto & name(i.first); - auto & value(i.second); - - auto setSubstituters = [&](Setting & res) { - if (name != res.name && res.aliases.count(name) == 0) - return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; - auto ss = tokenizeString(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else - warn("ignoring untrusted substituter '%s'", s); - res = subs; - return true; - }; - - try { - if (name == "ssh-auth-sock") // obsolete - ; - else if (trusted - || name == settings.buildTimeout.name - || name == "connect-timeout" - || (name == "builders" && value == "")) - settings.set(name, value); - else if (setSubstituters(settings.substituters)) - ; - else if (setSubstituters(settings.extraSubstituters)) - ; - else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); - } catch (UsageError & e) { - warn(e.what()); - } - } + // FIXME: use some setting in recursive mode. Will need to use + // non-global variables. + if (!recursive) + clientSettings.apply(trusted); logger->stopWork(); break; @@ -694,11 +728,12 @@ void processConnection( ref store, FdSource & from, FdSink & to, - bool trusted, + TrustedFlag trusted, + RecursiveFlag recursive, const std::string & userName, uid_t userId) { - MonitorFdHup monitor(from.fd); + auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; /* Exchange the greeting. */ unsigned int magic = readInt(from); @@ -712,7 +747,9 @@ void processConnection( auto tunnelLogger = new TunnelLogger(to, clientVersion); auto prevLogger = nix::logger; - logger = tunnelLogger; + // FIXME + if (!recursive) + logger = tunnelLogger; unsigned int opCount = 0; @@ -721,8 +758,10 @@ void processConnection( prevLogger->log(lvlDebug, fmt("%d operations", opCount)); }); - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + auto affinity = readInt(from); + setAffinityTo(affinity); + } readInt(from); // obsolete reserveSpace @@ -760,7 +799,7 @@ void processConnection( opCount++; try { - performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index fa26ba9c2..266932013 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -3,11 +3,15 @@ namespace nix::daemon { +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; +enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; + void processConnection( ref store, FdSource & from, FdSink & to, - bool trusted, + TrustedFlag trusted, + RecursiveFlag recursive, const std::string & userName, uid_t userId); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index da17ddcba..9314b6161 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -192,7 +192,7 @@ static void daemonLoop(char * * argv) closeOnExec(remote.get()); - bool trusted = false; + TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; @@ -205,7 +205,7 @@ static void daemonLoop(char * * argv) Strings allowedUsers = settings.allowedUsers; if (matchUser(user, group, trustedUsers)) - trusted = true; + trusted = Trusted; if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); @@ -239,7 +239,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); exit(0); }, options); @@ -321,7 +321,7 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(openUncachedStore(), from, to, true, "root", 0); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0); } } else { daemonLoop(argv); From c921074c1921d4bf392991a44c5644877f9c567a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 12:55:05 +0100 Subject: [PATCH 267/433] RestrictedStore: Implement addToStore() --- src/libstore/build.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2fd27cfa0..e8b23d396 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2721,7 +2721,14 @@ struct RestrictedStore : public LocalFSStore Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); + { throw Error("addToStore"); } + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0) override + { + next->addToStore(info, narSource, repair, checkSigs, accessor); + goal.addDependency(info.path); } Path addToStoreFromDump(const string & dump, const string & name, From 2af9561316606892ce0337939c01f3c0d43d75bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 13:23:58 +0100 Subject: [PATCH 268/433] Add a test for recursive Nix --- tests/local.mk | 3 ++- tests/recursive.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/recursive.sh diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..affdc3cc3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -30,7 +30,8 @@ nix_tests = \ search.sh \ nix-copy-ssh.sh \ post-hook.sh \ - function-trace.sh + function-trace.sh \ + recursive.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/recursive.sh b/tests/recursive.sh new file mode 100644 index 000000000..25204caf1 --- /dev/null +++ b/tests/recursive.sh @@ -0,0 +1,67 @@ +source common.sh + +clearStore + +export unreachable=$(nix add-to-store ./recursive.sh) + +nix build -o $TEST_ROOT/result -L '( + with import ./config.nix; + with import ; + mkDerivation { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + buildCommand = '\'\'' + mkdir $out + PATH=${nixBinDir}:$PATH + opts="--experimental-features nix-command" + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts add-to-store ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 3 ]] + + # Build a derivation. + nix $opts build -L '\''( + derivation { + name = "inner1"; + builder = builtins.getEnv "SHELL"; + system = builtins.getEnv "system"; + fnord = builtins.toFile "fnord" "fnord"; + args = [ "-c" "echo $fnord blaat > $out" ]; + } + )'\'' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + '\'\''; + }) +' + +[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] + +# Make sure the recursively created paths are in the closure. +nix path-info -r $TEST_ROOT/result | grep foobar +nix path-info -r $TEST_ROOT/result | grep fnord +nix path-info -r $TEST_ROOT/result | grep inner1 From c119ab9db0edf65379593883c5ed5253549ebbd0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 13:48:58 +0100 Subject: [PATCH 269/433] Enable recursive Nix using a feature Derivations that want to use recursion should now set requiredSystemFeatures = [ "recursive-nix" ]; to make the daemon socket appear. Also, Nix should be configured with "experimental-features = recursive-nix". --- src/libstore/build.cc | 5 ++++- src/libstore/globals.cc | 2 +- tests/recursive.sh | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e8b23d396..e77512ca4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2248,7 +2248,8 @@ void DerivationGoal::startBuilder() /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ - startDaemon(); + if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) + startDaemon(); /* Run the builder. */ printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); @@ -2832,6 +2833,8 @@ struct RestrictedStore : public LocalFSStore void DerivationGoal::startDaemon() { + settings.requireExperimentalFeature("recursive-nix"); + Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 249c36673..042aaea95 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -95,7 +95,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel"}; + StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) diff --git a/tests/recursive.sh b/tests/recursive.sh index 25204caf1..62896c015 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -4,7 +4,7 @@ clearStore export unreachable=$(nix add-to-store ./recursive.sh) -nix build -o $TEST_ROOT/result -L '( +nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( with import ./config.nix; with import ; mkDerivation { @@ -15,6 +15,8 @@ nix build -o $TEST_ROOT/result -L '( # Note: this is a string without context. unreachable = builtins.getEnv "unreachable"; + requiredSystemFeatures = [ "recursive-nix" ]; + buildCommand = '\'\'' mkdir $out PATH=${nixBinDir}:$PATH From 69326f3637f1560407711838e7298d736274ffd4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 14:27:28 +0100 Subject: [PATCH 270/433] Recursive Nix: Handle concurrent client connections --- src/libstore/build.cc | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e77512ca4..63557b560 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -895,6 +895,9 @@ private: /* The daemon main thread. */ std::thread daemonThread; + /* The daemon worker threads. */ + std::vector daemonWorkerThreads; + /* Paths that were added via recursive Nix calls. */ PathSet addedPaths; @@ -2875,17 +2878,19 @@ void DerivationGoal::startDaemon() debug("received daemon connection"); - // FIXME: process on a separate thread. - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, "nobody", 65535); - } catch (SysError &) { - ignoreException(); - } + auto workerThread = std::thread([this, store, remote{std::move(remote)}]() { + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + debug("terminated daemon connection"); + } catch (SysError &) { + ignoreException(); + } + }); - debug("terminated daemon connection"); + daemonWorkerThreads.push_back(std::move(workerThread)); } debug("daemon shutting down"); @@ -2901,6 +2906,12 @@ void DerivationGoal::stopDaemon() if (daemonThread.joinable()) daemonThread.join(); + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + daemonSocket = -1; } From 7614a127a0ef52868c35fa9d6ae6398b7297d408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 10:35:31 +0100 Subject: [PATCH 271/433] Fix binaryTarball test --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 0b74de390..c5bee782a 100644 --- a/release.nix +++ b/release.nix @@ -260,7 +260,7 @@ let x86_64-linux = "${build.x86_64-linux}"; } EOF - su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix' + su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix' (! [ -L /home/alice/.profile-1-link ]) su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' From 35732a95bcdc0a4b4492845205e6283fcc88fd0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 10:36:06 +0100 Subject: [PATCH 272/433] Disable the evalNixpkgs test It constantly OOMs. https://hydra.nixos.org/build/105784912 --- release.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index c5bee782a..b137f1d2d 100644 --- a/release.nix +++ b/release.nix @@ -269,6 +269,7 @@ let umount /nix ''); # */ + /* tests.evalNixpkgs = import (nixpkgs + "/pkgs/top-level/make-tarball.nix") { inherit nixpkgs; @@ -276,6 +277,7 @@ let nix = build.x86_64-linux; officialRelease = false; }; + */ tests.evalNixOS = pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } @@ -324,7 +326,7 @@ let tests.remoteBuilds tests.nix-copy-closure tests.binaryTarball - tests.evalNixpkgs + #tests.evalNixpkgs tests.evalNixOS installerScript ]; From 39a2e166dd316cf3bf248cb109dde3a95d753541 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 16:53:02 +0100 Subject: [PATCH 273/433] Cleanup --- src/libstore/build.cc | 2 +- src/libutil/util.cc | 2 +- src/libutil/xml-writer.cc | 4 ++-- src/nix-daemon/nix-daemon.cc | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ada1cf8ec..c82f60748 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -23,8 +23,8 @@ #include #include #include +#include -#include #include #include #include diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 34c1d96dc..79716c5a7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,7 +18,6 @@ #include #include -#include #include #include #include diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index e5cc2e9fc..68857e34d 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -1,10 +1,10 @@ -#include +#include #include "xml-writer.hh" namespace nix { - + XMLWriter::XMLWriter(bool indent, std::ostream & output) : output(output), indent(indent) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index da17ddcba..65ae88faf 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -10,8 +10,9 @@ #include "daemon.hh" #include - +#include #include + #include #include #include @@ -23,7 +24,6 @@ #include #include #include -#include #if __APPLE__ || __FreeBSD__ #include From 52ffe2797a6a943bac968e92ad443a567dfdf80e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 7 Nov 2019 00:11:57 +0100 Subject: [PATCH 274/433] doc: Document `--dry-run` option for `nix-build` --- doc/manual/command-ref/nix-build.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml index 40fe7a43f..c1b783c87 100644 --- a/doc/manual/command-ref/nix-build.xml +++ b/doc/manual/command-ref/nix-build.xml @@ -30,6 +30,7 @@ attrPath + @@ -98,6 +99,10 @@ also . + + Show what store paths would be built or downloaded + + / outlink From 5ff4d77f5585f9aad068fcc473393095e0640ae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 16:54:18 +0100 Subject: [PATCH 275/433] Precompile headers This cuts 'make install -j6' on my laptop from 170s to 134s. --- Makefile | 4 +-- configure.ac | 1 - mk/lib.mk | 1 + mk/patterns.mk | 4 +-- mk/precompiled-headers.mk | 42 +++++++++++++++++++++++ precompiled-headers.h | 71 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 mk/precompiled-headers.mk create mode 100644 precompiled-headers.h diff --git a/Makefile b/Makefile index 9ac82fda6..c441a933e 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ makefiles = \ tests/local.mk \ tests/plugins/local.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h - -include Makefile.config include mk/lib.mk + +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 diff --git a/configure.ac b/configure.ac index 4a6eaac8a..c67f1e739 100644 --- a/configure.ac +++ b/configure.ac @@ -57,7 +57,6 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) AC_CHECK_TOOL([AR], [ar]) diff --git a/mk/lib.mk b/mk/lib.mk index 1da51d879..b0a36cf8d 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -89,6 +89,7 @@ include mk/jars.mk include mk/patterns.mk include mk/templates.mk include mk/tests.mk +include mk/precompiled-headers.mk # Include all sub-Makefiles. diff --git a/mk/patterns.mk b/mk/patterns.mk index 3219d9629..7319f4cdd 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk new file mode 100644 index 000000000..779389b26 --- /dev/null +++ b/mk/precompiled-headers.mk @@ -0,0 +1,42 @@ +PRECOMPILE_HEADERS ?= 1 + +print-var-help += \ + echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build"; + +GCH = $(buildprefix)precompiled-headers.h.gch + +$(GCH): precompiled-headers.h + @rm -f $@ + @mkdir -p "$(dir $@)" + $(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS) + +PCH = $(buildprefix)precompiled-headers.h.pch + +$(PCH): precompiled-headers.h + @rm -f $@ + @mkdir -p "$(dir $@)" + $(trace-gen) $(CXX) -x c++-header -o $@ $^ $(GLOBAL_CXXFLAGS) + +clean-files += $(GCH) $(PCH) + +ifeq ($(PRECOMPILE_HEADERS), 1) + + ifeq ($(CXX), g++) + + GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(GCH) + + else ifeq ($(CXX), clang++) + + GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch + + GLOBAL_ORDER_AFTER += $(PCH) + + else + + $(error Don't know how to precompile headers on $(CXX)) + + endif + +endif diff --git a/precompiled-headers.h b/precompiled-headers.h new file mode 100644 index 000000000..c5790bec4 --- /dev/null +++ b/precompiled-headers.h @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.hh" +#include "args.hh" From f5b7991e59d8c9ae38fb4c9074a5847942caab73 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 16:55:37 +0100 Subject: [PATCH 276/433] Revert "autoconf: Allow overriding CFLAGS/CXXFLAGS from outside." This reverts commit 717e821b99797845e1bef47d862f8cb0fb69cfc9. It's much more convenient to do 'make OPTIMIZE=0'. --- Makefile | 8 ++++++++ configure.ac | 6 ++---- perl/Makefile | 7 +++++++ perl/configure.ac | 6 ++---- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c441a933e..fe397e018 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,14 @@ makefiles = \ -include Makefile.config +OPTIMIZE = 1 + +ifeq ($(OPTIMIZE), 1) + GLOBAL_CXXFLAGS += -O3 +else + GLOBAL_CXXFLAGS += -O0 +endif + include mk/lib.mk GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 diff --git a/configure.ac b/configure.ac index c67f1e739..26f16b846 100644 --- a/configure.ac +++ b/configure.ac @@ -50,10 +50,8 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Set default flags for nix (as per AC_PROG_CC/CXX docs), -# while still allowing the user to override them from the command line. -: ${CFLAGS="-O3"} -: ${CXXFLAGS="-O3"} +CFLAGS= +CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP diff --git a/perl/Makefile b/perl/Makefile index f36f5d0e9..284c75022 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -4,4 +4,11 @@ GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config +OPTIMIZE = 1 + +ifeq ($(OPTIMIZE), 1) + GLOBAL_CFLAGS += -O3 + GLOBAL_CXXFLAGS += -O3 +endif + include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index e8e3610a8..966700695 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -2,10 +2,8 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) -# Set default flags for nix (as per AC_PROG_CC/CXX docs), -# while still allowing the user to override them from the command line. -: ${CFLAGS="-O3"} -: ${CXXFLAGS="-O3"} +CFLAGS= +CXXFLAGS= AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 From 04bf9acd22c5e3f399f66d7edb0863e8394e542e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Nov 2019 17:30:48 +0100 Subject: [PATCH 277/433] Remove #include --- src/libstore/download.cc | 4 ++++ src/libstore/download.hh | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cdf56e09d..8fe278d02 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -34,6 +34,10 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); +CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) + : uri(uri), ttl(settings.tarballTtl) +{ } + std::string resolveUri(const std::string & uri) { if (uri.compare(0, 8, "channel:") == 0) diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 68565bf46..5a131c704 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "hash.hh" -#include "globals.hh" +#include "config.hh" #include #include @@ -71,10 +71,10 @@ struct CachedDownloadRequest bool unpack = false; std::string name; Hash expectedHash; - unsigned int ttl = settings.tarballTtl; + unsigned int ttl; - CachedDownloadRequest(const std::string & uri) - : uri(uri) { } + CachedDownloadRequest(const std::string & uri); + CachedDownloadRequest() = delete; }; struct CachedDownloadResult From 99af822004418b25b6d04eca67949c62e770c16b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Nov 2019 10:14:00 +0100 Subject: [PATCH 278/433] Disable the evalNixOS test It also OOMs. https://hydra.nixos.org/build/105942679 --- release.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index b137f1d2d..61ca4323c 100644 --- a/release.nix +++ b/release.nix @@ -277,7 +277,6 @@ let nix = build.x86_64-linux; officialRelease = false; }; - */ tests.evalNixOS = pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } @@ -289,6 +288,7 @@ let touch $out ''; + */ installerScript = @@ -327,7 +327,7 @@ let tests.nix-copy-closure tests.binaryTarball #tests.evalNixpkgs - tests.evalNixOS + #tests.evalNixOS installerScript ]; }; From 6d2605500fd99091cea2985499957a0aace0d81b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Nov 2019 11:53:28 +0100 Subject: [PATCH 279/433] Fix macOS build --- precompiled-headers.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/precompiled-headers.h b/precompiled-headers.h index c5790bec4..e0d885b23 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -43,23 +43,13 @@ #include #include #include -#include #include -#include #include -#include #include -#include -#include -#include -#include -#include #include #include #include -#include #include -#include #include #include #include From e5bf81256c8a7ee9d725d4093853c76e0a562c66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Nov 2019 12:18:37 +0100 Subject: [PATCH 280/433] Fix Perl bindings --- Makefile | 1 + mk/lib.mk | 1 - perl/Makefile | 3 ++- perl/configure.ac | 14 +++++++------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index fe397e018..866c0961e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ makefiles = \ + mk/precompiled-headers.mk \ local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ diff --git a/mk/lib.mk b/mk/lib.mk index b0a36cf8d..1da51d879 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -89,7 +89,6 @@ include mk/jars.mk include mk/patterns.mk include mk/templates.mk include mk/tests.mk -include mk/precompiled-headers.mk # Include all sub-Makefiles. diff --git a/perl/Makefile b/perl/Makefile index 284c75022..7ddb0cf69 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -7,8 +7,9 @@ GLOBAL_CXXFLAGS += -g -Wall OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 GLOBAL_CXXFLAGS += -O3 +else + GLOBAL_CXXFLAGS += -O0 endif include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index 966700695..c3769e142 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -6,7 +6,6 @@ CFLAGS= CXXFLAGS= AC_PROG_CC AC_PROG_CXX -AX_CXX_COMPILE_STDCXX_11 # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE @@ -69,14 +68,15 @@ AC_SUBST(perlFlags) PKG_CHECK_MODULES([NIX], [nix-store]) -NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate]) +NEED_PROG([NIX], [nix]) # Get nix configure values -nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixBinDir | tr -d \") -nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixLibexecDir | tr -d \") -nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixLocalstateDir | tr -d \") -nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixSysconfDir | tr -d \") -nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixStoreDir | tr -d \") +export NIX_REMOTE=daemon +nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixBinDir) +nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLibexecDir) +nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLocalstateDir) +nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixSysconfDir) +nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixStoreDir) AC_SUBST(nixbindir) AC_SUBST(nixlibexecdir) AC_SUBST(nixlocalstatedir) From 4145cd2da002e1bd8affa0392c80118eabe58e3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 20:12:23 +0200 Subject: [PATCH 281/433] Use upstream nlohmann_json --- release-common.nix | 1 + release.nix | 2 +- src/nlohmann/json.hpp | 20406 ---------------------------------------- 3 files changed, 2 insertions(+), 20407 deletions(-) delete mode 100644 src/nlohmann/json.hpp diff --git a/release-common.nix b/release-common.nix index 11944a631..63f39f005 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,6 +50,7 @@ rec { bzip2 xz brotli editline openssl pkgconfig sqlite boehmgc boost + nlohmann_json # Tests git diff --git a/release.nix b/release.nix index 61ca4323c..1af6424dd 100644 --- a/release.nix +++ b/release.nix @@ -200,7 +200,7 @@ let doInstallCheck = true; - lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ]; + lcovFilter = [ "*/boost/*" "*-tab.*" ]; # We call `dot', and even though we just use it to # syntax-check generated dot files, it still requires some diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp deleted file mode 100644 index c9af0bed3..000000000 --- a/src/nlohmann/json.hpp +++ /dev/null @@ -1,20406 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP - -#define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 - -#include // all_of, find, for_each -#include // assert -#include // and, not, or -#include // nullptr_t, ptrdiff_t, size_t -#include // hash, less -#include // initializer_list -#include // istream, ostream -#include // random_access_iterator_tag -#include // accumulate -#include // string, stoi, to_string -#include // declval, forward, move, pair, swap - -// #include -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string -#include // vector - -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; - -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer> -class basic_json; - -/*! -@brief JSON Pointer - -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. - -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - -@since version 2.0.0 -*/ -template -class json_pointer; - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; -} // namespace nlohmann - -#endif - -// #include - - -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them - -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" -#endif - -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) -#else - #define JSON_DEPRECATED -#endif - -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) -#else - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) -#endif - -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER -#endif - -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif - -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif - -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } - -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. - -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> - -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json - -// #include - - -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type - -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; - -template -using uncvref_t = typename std::remove_cv::type>::type; - -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; - -template -struct merge_and_renumber; - -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; - -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; - -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; - -template -using index_sequence_for = make_index_sequence; - -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; - -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; - -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval - -// #include - -// #include - - -#include // random_access_iterator_tag - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; - -template -struct iterator_traits::value>> -{ - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; -}; -} -} - -// #include - -// #include - - -#include - -// #include - - -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ -namespace detail -{ -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - void operator=(nonesuch const&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template