From 15833516a4bad0a4ae7786293b22df4bf650aa80 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 5 Feb 2019 16:42:45 -0500 Subject: [PATCH 001/725] Add armv6l-linux & armv7l-linux as cross jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a cheap way to get 32-bit ARM working. We don’t support it officially but lots of people have raspberry pis and similar hardware they want to install the Nix package manager on. --- release.nix | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/release.nix b/release.nix index 271645067..0ec742906 100644 --- a/release.nix +++ b/release.nix @@ -2,6 +2,7 @@ , nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, crossSystems ? [ "armv6l-linux" "armv7l-linux" ] }: let @@ -53,11 +54,12 @@ let }; - build = pkgs.lib.genAttrs systems (system: + build = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; with import ./release-common.nix { inherit pkgs; }; @@ -89,9 +91,12 @@ let }); - perlBindings = pkgs.lib.genAttrs systems (system: + perlBindings = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - let pkgs = import nixpkgs { inherit system; }; in with pkgs; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; releaseTools.nixBuild { name = "nix-perl"; @@ -112,9 +117,12 @@ let }); - binaryTarball = pkgs.lib.genAttrs systems (system: + binaryTarball = pkgs.lib.genAttrs (systems ++ crossSystems) (system: - with import nixpkgs { inherit system; }; + let pkgs = if builtins.elem system systems + then import nixpkgs { inherit system; } + else import nixpkgs { crossSystem = { inherit system; }; }; + in with pkgs; let toplevel = builtins.getAttr system jobs.build; From 1996af425ac8ddea1e8a591650e7d0caba2aa201 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 21:43:47 -0500 Subject: [PATCH 002/725] Use buildPackages for native dependencies Unfortunately, releaseTools.nixBuild does not separate native and non-native build inputs. As an alternative, we can just use buildPackages to get the native version of some packages like: - pkgconfig - git - curl - utillinux --- release-common.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/release-common.nix b/release-common.nix index 4c5565985..f2aa57c2c 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,14 +50,16 @@ rec { buildDeps = [ curl bzip2 xz brotli editline - openssl pkgconfig sqlite boehmgc + openssl sqlite boehmgc boost + buildPackages.pkgconfig + # Tests - git - mercurial + buildPackages.git + buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { From e9072ded9749ab00cc397980e8a26f83d341efc0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 22:43:28 -0500 Subject: [PATCH 003/725] Use nativeBuildInputs --- release-common.nix | 16 +++++++++------- release.nix | 3 +++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/release-common.nix b/release-common.nix index f2aa57c2c..707d36f95 100644 --- a/release-common.nix +++ b/release-common.nix @@ -47,19 +47,21 @@ rec { autoreconfHook ]; + nativeBuildDeps = + [ buildPackages.pkgconfig + + # Tests + buildPackages.git + buildPackages.mercurial + ] ++ lib.optional stdenv.isLinux buildPackages.utillinuxMinimal; + buildDeps = [ curl bzip2 xz brotli editline openssl sqlite boehmgc boost - - buildPackages.pkgconfig - - # Tests - buildPackages.git - buildPackages.mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp buildPackages.utillinuxMinimal] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { diff --git a/release.nix b/release.nix index 0ec742906..9843efa29 100644 --- a/release.nix +++ b/release.nix @@ -24,6 +24,7 @@ let src = nix; inherit officialRelease; + nativeBuildInputs = nativeBuildDeps; buildInputs = tarballDeps ++ buildDeps; configureFlags = "--enable-gc"; @@ -67,6 +68,7 @@ let name = "nix"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; preConfigure = @@ -199,6 +201,7 @@ let name = "nix-build"; src = tarball; + nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps; dontInstall = false; From f6ea56dfac1f4df45a5fa9e2801bc632dee9eff7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 6 Feb 2019 23:04:40 -0500 Subject: [PATCH 004/725] Get shellcheck from buildPackages --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 9843efa29..06db7bdc0 100644 --- a/release.nix +++ b/release.nix @@ -133,7 +133,7 @@ let in runCommand "nix-binary-tarball-${version}" - { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + { nativeBuildInputs = lib.optional (system != "aarch64-linux") buildPackages.shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' From 59f2dd8e8da1f82aa9e29e30ba1df643434a9254 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 6 Oct 2020 20:08:51 +0200 Subject: [PATCH 005/725] libfetchers/github: allow slashes in refs Refs #4061 --- src/libfetchers/github.cc | 20 +++++++++++++++++--- src/libutil/url-parts.hh | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..3d1cc15e2 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme std::optional ref; std::optional host_url; - if (path.size() == 2) { - } else if (path.size() == 3) { + auto size = path.size(); + if (size == 3) { if (std::regex_match(path[2], revRegex)) rev = Hash::parseAny(path[2], htSHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else + } else if (size > 3) { + std::string rs; + for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { + rs += *i; + if (std::next(i) != path.end()) { + rs += "/"; + } + } + + if (std::regex_match(rs, refRegex)) { + ref = rs; + } else { + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + } + } else if (size < 2) throw BadURL("URL '%s' is invalid", url.url); for (auto &[name, value] : url.query) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 68be15cb0..e0e2809fd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -22,7 +22,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; // A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check extern std::regex refRegex; // Instead of defining what a good Git Ref is, we define what a bad Git Ref is From 107c91f5fe6248548c292d28d0ad53c0de7ceeba Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 16:48:35 -0700 Subject: [PATCH 006/725] auto-call error --- src/libexpr/eval.cc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6366050c..e52e8dcf2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,7 +1370,28 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + An 'auto-call' is when a nix expression is evaluated without any external arguments. If that + nix expression is a function, and that function's arguments all have default values, then all is well. + + But if the function arguments don't have default values, then evaluation fails. + + The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... + + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + + See this nix pill for more information re callPackage format: + https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 6c2933a8d72f9328a2931a8166439bed96b80f24 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:04:52 -0700 Subject: [PATCH 007/725] add position --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e52e8dcf2..11f2e31ce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1370,11 +1370,11 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default values, then all is well. - But if the function arguments don't have default values, then evaluation fails. + But if the function arguments don't have default values, evaluation fails. The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. From 9f2b25ce55e38fc5772fcb65bd98651255f6a49b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 17:17:47 -0700 Subject: [PATCH 008/725] remove unused ftn; reformat line breaks --- src/libexpr/eval.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 11f2e31ce..0446376f3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -636,11 +636,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(s, s1); -} - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ @@ -1371,12 +1366,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. If that - nix expression is a function, and that function's arguments all have default values, then all is well. + An 'auto-call' is when a nix expression is evaluated without any external arguments. + If that nix expression is a function, and that function's arguments all have default + values, then all is well. But if the function arguments don't have default values, evaluation fails. - The classic case for this error is evaluating a nix file with nix-build that expects to be evaluated by callPackage. + The classic case for this error is evaluating a nix file with nix-build that expects + to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. { stdenv }: From d8ef423a189b95f9f6e0a385a0f4474bebf297ef Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 9 Nov 2020 19:16:50 -0700 Subject: [PATCH 009/725] error message formatting --- src/libexpr/eval.cc | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0446376f3..2b697882b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1366,29 +1366,30 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) actualArgs->attrs->push_back(*j); } else if (!i.def) { throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') - An 'auto-call' is when a nix expression is evaluated without any external arguments. - If that nix expression is a function, and that function's arguments all have default - values, then all is well. - But if the function arguments don't have default values, evaluation fails. +An 'auto-call' is when a nix expression is evaluated without any external arguments. +If that nix expression is a function, and that function's arguments all have default +values, then all is well. - The classic case for this error is evaluating a nix file with nix-build that expects - to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... +But if the function arguments don't have default values, evaluation fails. - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... +The classic case for this error is evaluating a nix file with nix-build that expects +to be evaluated by callPackage. + # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. + # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. + { stdenv }: + stdenv.mkDerivation { + ... - See this nix pill for more information re callPackage format: - https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); + # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. + let + nixpkgs = import {}; + in + nixpkgs.stdenv.mkDerivation { + ... + +More about callPackage: +https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From 7d9037035ef8bfe0b7ae00a9e3b139ae83ec8b21 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:21:26 -0700 Subject: [PATCH 010/725] usage example location --- src/libutil/error.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..d42781311 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -38,7 +38,7 @@ namespace nix { ErrorInfo structs are sent to the logger as part of an exception, or directly with the logError or logWarning macros. - See the error-demo.cc program for usage examples. + See libutil/tests/logging.cc for usage examples. */ From 3edfe6090e9e15b205c21b19530607cbdcbbbe7a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 09:29:32 -0700 Subject: [PATCH 011/725] missing argument error --- src/libexpr/eval.cc | 12 ++++++++++-- src/libexpr/nixexpr.hh | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2b697882b..cf9f6c543 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -660,6 +660,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * }); } +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +{ + throw MissingArgumentError({ + .hint = hintfmt(s, s1), + .errPos = pos + }); +} + LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) { e.addTrace(std::nullopt, s, s2); @@ -1365,7 +1373,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwUndefinedVarError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') An 'auto-call' is when a nix expression is evaluated without any external arguments. If that nix expression is a function, and that function's arguments all have default @@ -1373,7 +1381,7 @@ values, then all is well. But if the function arguments don't have default values, evaluation fails. -The classic case for this error is evaluating a nix file with nix-build that expects +The classic case for this error is evaluating a nix file that expects to be evaluated by callPackage. # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..bf2cd1f15 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); +MakeError(MissingArgumentError, Error); MakeError(RestrictedPathError, Error); From 8895fa70a4b05ddebbb5a23ea96464d5e01345fb Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:05:21 -0700 Subject: [PATCH 012/725] pare down the error message --- src/libexpr/eval.cc | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cf9f6c543..540bfcf7b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1373,31 +1373,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot auto-call a function that has an argument without a default value ('%1%') + throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -An 'auto-call' is when a nix expression is evaluated without any external arguments. -If that nix expression is a function, and that function's arguments all have default -values, then all is well. +Nix attempted to evaluate a function as a top level expression; in this case it must have all its +arguments supplied either by default values, or passed explicitly with --arg or --argstr. -But if the function arguments don't have default values, evaluation fails. +https://nixos.org/manual/nix/stable/#ss-functions)", i.name); -The classic case for this error is evaluating a nix file that expects -to be evaluated by callPackage. - # in 'callPackage' format: expression is a function that takes an argument 'stdenv'. - # callPackage would implicitly pull 'stdenv' from nixpkgs, then call this function. - { stdenv }: - stdenv.mkDerivation { - ... - - # in 'auto-call' format: nixpkgs is imported explicitly, and used directly. - let - nixpkgs = import {}; - in - nixpkgs.stdenv.mkDerivation { - ... - -More about callPackage: -https://nixos.org/guides/nix-pills/callpackage-design-pattern.html)", i.name); } } } From b327de9c2d5182e5814b2e956631b8794b45999b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 11 Nov 2020 11:09:59 -0700 Subject: [PATCH 013/725] change message --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 540bfcf7b..3667ee6ba 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1375,7 +1375,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -Nix attempted to evaluate a function as a top level expression; in this case it must have all its +nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with --arg or --argstr. https://nixos.org/manual/nix/stable/#ss-functions)", i.name); From 437189e446e16399d347e4430c4d115b4cf2ddf1 Mon Sep 17 00:00:00 2001 From: Lily Ballard Date: Tue, 24 Nov 2020 14:12:32 -0800 Subject: [PATCH 014/725] Escape filename given to nix-shell in shebang mode This prevents spaces or other metacharacters from causing nix-shell to execute the wrong path. Fixes #4229. --- src/nix-build/nix-build.cc | 4 ++-- tests/nix-shell.sh | 18 +++++++++++++++++- tests/shell.nix | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 74fafd426..38048da52 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } } diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 7b2be650a..4775bafb9 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -47,6 +47,14 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode again with metacharacters in the filename. +# First word of filename is chosen to not match any file in the test root. +sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + # Test nix-shell shebang mode for ruby # This uses a fake interpreter that returns the arguments passed # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. @@ -54,7 +62,15 @@ sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shel chmod a+rx $TEST_ROOT/shell.shebang.rb output=$($TEST_ROOT/shell.shebang.rb abc ruby) -[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] + +# Test nix-shell shebang mode for ruby again with metacharacters in the filename. +# Note: fake interpreter only space-separates args without adding escapes to its output. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/shell.nix b/tests/shell.nix index 6ce59b416..24ebcc04c 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -50,7 +50,7 @@ let pkgs = rec { # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" {} '' mkdir -p $out/bin - echo 'printf -- "$*"' > $out/bin/ruby + echo 'printf %s "$*"' > $out/bin/ruby chmod a+rx $out/bin/ruby ''; From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: [PATCH 015/725] Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: [PATCH 016/725] Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: [PATCH 017/725] Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin") From 29bd63e9907cabc5643aaa3f570b9ff5b2d88268 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 19:55:21 +0000 Subject: [PATCH 018/725] Test nix-instantiate with binary cache store Trying to make sure it work with obscurers stores. --- tests/binary-cache.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..8f1c6f14d 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,15 +1,20 @@ source common.sh +# We can produce drvs directly into the binary cache clearStore -clearCache +clearCacheCache +nix-instantiate --store "file://$cacheDir" dependencies.nix # Create the binary cache. +clearStore +clearCache outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to file://$cacheDir $outPath -basicTests() { +basicDownloadTests() { + # No uploading tests bcause upload with force HTTP doesn't work. # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. @@ -44,12 +49,12 @@ basicTests() { # Test LocalBinaryCacheStore. -basicTests +basicDownloadTests # Test HttpBinaryCacheStore. export _NIX_FORCE_HTTP=1 -basicTests +basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. From 57062179ce36e35715284d2ef570f8cb0b90198d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 16:05:09 +0000 Subject: [PATCH 019/725] Move some PKI stuff from LocalStore to Store --- src/libstore/local-store.cc | 9 --------- src/libstore/local-store.hh | 12 ------------ src/libstore/misc.cc | 9 +++++++++ src/libstore/store-api.hh | 13 +++++++++++++ 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..1eb2dec75 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1092,15 +1092,6 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } -const PublicKeys & LocalStore::getPublicKeys() -{ - auto state(_state.lock()); - if (!state->publicKeys) - state->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *state->publicKeys; -} - - void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..d97645058 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,10 +35,6 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const std::string name() override { return "Local Store"; } }; @@ -75,8 +71,6 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); - - std::unique_ptr publicKeys; }; Sync _state; @@ -94,12 +88,6 @@ public: const Path tempRootsDir; const Path fnTempRoots; -private: - - const PublicKeys & getPublicKeys(); - -public: - // Hack for build-remote.cc. PathSet locksHeld; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..0d4190a56 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,4 +282,13 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } +const PublicKeys & Store::getPublicKeys() +{ + auto cryptoState(_cryptoState.lock()); + if (!cryptoState->publicKeys) + cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *cryptoState->publicKeys; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..e3de6db17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,6 +189,10 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + Setting requireSigs{this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -710,11 +714,20 @@ public: return toRealPath(printStorePath(storePath)); } + const PublicKeys & getPublicKeys(); + virtual void createUser(const std::string & userName, uid_t userId) { } protected: + struct CryptoState + { + std::unique_ptr publicKeys; + }; + + Sync _cryptoState; + Stats stats; /* Unsupported methods. */ From 12f7a1f65becfe3b036d0f840ee4a05f2f1f857c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:07:28 +0000 Subject: [PATCH 020/725] build-remote no longer requires local store be local --- src/build-remote/build-remote.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8348d8c91..350bd6cef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -71,11 +71,15 @@ static int main_build_remote(int argc, char * * argv) initPlugins(); - auto store = openStore().cast(); + auto store = openStore(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = store->stateDir + "/current-load"; + currentLoad = "/current-load"; + if (auto localStore = store.dynamic_pointer_cast()) + currentLoad = std::string { localStore->stateDir } + currentLoad; + else + currentLoad = settings.nixStateDir + currentLoad; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; @@ -288,8 +292,9 @@ connected: if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - for (auto & i : missing) - store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ + if (auto localStore = store.dynamic_pointer_cast()) + for (auto & i : missing) + localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } From 450c3500f1e3fb619636c0a29d65300020f99d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:36:52 +0000 Subject: [PATCH 021/725] Crudely make worker only provide a Store, not LocalStore We downcast in a few places, this will be refactored to be better later. --- src/libstore/build/derivation-goal.cc | 66 ++++++++++++++++++--------- src/libstore/build/worker.cc | 6 ++- src/libstore/build/worker.hh | 9 ++-- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 47d11dc53..de32f60db 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -848,14 +848,16 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; + if (auto localStore = dynamic_cast(&worker.store)) { + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } #endif deleteTmpDir(false); @@ -1215,12 +1217,15 @@ void DerivationGoal::startBuilder() useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif + if (auto localStoreP = dynamic_cast(&worker.store)) { + auto & localStore = *localStoreP; + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } } /* Create a temporary directory where the build will take @@ -2182,7 +2187,8 @@ void DerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; + if (auto localStore = dynamic_cast(&worker.store)) + params["root"] = localStore->rootDir; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -3246,7 +3252,13 @@ void DerivationGoal::registerOutputs() } } + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { @@ -3271,8 +3283,8 @@ void DerivationGoal::registerOutputs() /* Since we verified the build, it's now ultimately trusted. */ if (!oldInfo.ultimate) { oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3288,13 +3300,13 @@ void DerivationGoal::registerOutputs() } if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() + localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(newInfo.path); } newInfo.deriver = drvPath; newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); + localStore.signPathInfo(newInfo); finish(newInfo.path); @@ -3302,7 +3314,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({{newInfo.path, newInfo}}); + localStore.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3375,11 +3387,16 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ { + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { infos2.insert_or_assign(newInfo.path, newInfo); } - worker.store.registerValidPaths(infos2); + localStore.registerValidPaths(infos2); } /* In case of a fixed-output derivation hash mismatch, throw an @@ -3577,7 +3594,12 @@ Path DerivationGoal::openLogFile() auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); + Path logDir; + if (auto localStore = dynamic_cast(&worker.store)) + logDir = localStore->logDir; + else + logDir = settings.nixLogDir; + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); createDirs(dir); Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..a9575fb0f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -8,7 +8,7 @@ namespace nix { -Worker::Worker(LocalStore & store) +Worker::Worker(Store & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) @@ -229,7 +229,9 @@ void Worker::run(const Goals & _topGoals) checkInterrupt(); - store.autoGC(false); + // TODO GC interface? + if (auto localStore = dynamic_cast(&store)) + localStore->autoGC(false); /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index bf8cc4586..82e711191 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,9 +2,12 @@ #include "types.hh" #include "lock.hh" -#include "local-store.hh" +#include "store-api.hh" #include "goal.hh" +#include +#include + namespace nix { /* Forward definition. */ @@ -102,7 +105,7 @@ public: /* Set if at least one derivation is not deterministic in check mode. */ bool checkMismatch; - LocalStore & store; + Store & store; std::unique_ptr hook; @@ -124,7 +127,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(LocalStore & store); + Worker(Store & store); ~Worker(); /* Make a goal (with caching). */ From 85f2e9e8fa4f7452a05cfffc901d118a7c861d0a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:54:57 +0000 Subject: [PATCH 022/725] Expose schedule entrypoints to all stores Remote stores still override so the other end schedules. --- src/libstore/binary-cache-store.hh | 7 ------ .../{local-store-build.cc => entry-points.cc} | 6 ++--- src/libstore/dummy-store.cc | 7 ------ src/libstore/local-store.hh | 9 -------- src/libstore/store-api.cc | 23 ------------------- src/libstore/store-api.hh | 6 ++--- 6 files changed, 6 insertions(+), 52 deletions(-) rename src/libstore/build/{local-store-build.cc => entry-points.cc} (91%) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 443a53cac..c2163166c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -108,13 +108,6 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - ref getFSAccessor() override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/entry-points.cc similarity index 91% rename from src/libstore/build/local-store-build.cc rename to src/libstore/build/entry-points.cc index c91cda2fd..9f97d40ba 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/entry-points.cc @@ -5,7 +5,7 @@ namespace nix { -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); @@ -43,7 +43,7 @@ void LocalStore::buildPaths(const std::vector & drvPaths, } } -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, +BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); @@ -63,7 +63,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe } -void LocalStore::ensurePath(const StorePath & path) +void Store::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3c7caf8f2..8f26af685 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -55,13 +55,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - std::optional queryRealisation(const DrvOutput&) override { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d97645058..aa5de31f0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -133,15 +133,6 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void buildPaths( - const std::vector & paths, - BuildMode buildMode) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; - void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..f12a564a1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -747,29 +747,6 @@ const Store::Stats & Store::getStats() } -void Store::buildPaths(const std::vector & paths, BuildMode buildMode) -{ - StorePathSet paths2; - - for (auto & path : paths) { - if (path.path.isDerivation()) { - auto outPaths = queryPartialDerivationOutputMap(path.path); - for (auto & outputName : path.outputs) { - auto currentOutputPathIter = outPaths.find(outputName); - if (currentOutputPathIter == outPaths.end() || - !currentOutputPathIter->second || - !isValidPath(*currentOutputPathIter->second)) - unsupported("buildPaths"); - } - } else - paths2.insert(path.path); - } - - if (queryValidPaths(paths2).size() != paths2.size()) - unsupported("buildPaths"); -} - - void copyStorePath(ref srcStore, ref dstStore, const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3de6db17..4db980fe9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -523,17 +523,17 @@ public: explicitly choosing to allow it). */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) = 0; + BuildMode buildMode = bmNormal); /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(const StorePath & path); /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ virtual void addTempRoot(const StorePath & path) - { unsupported("addTempRoot"); } + { warn("not creating temp root, store doesn't support GC"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed From fed123724679de89d3f56a4c01b5c4c96f93e584 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 19:55:21 +0000 Subject: [PATCH 023/725] Test nix-build with non-local-store --store Just a few small things needed fixing! --- src/libstore/build/derivation-goal.cc | 20 +++++++++++++++++--- tests/binary-cache-build-remote.sh | 13 +++++++++++++ tests/local.mk | 4 +++- 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/binary-cache-build-remote.sh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de32f60db..17f39a86e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -592,9 +592,17 @@ void DerivationGoal::tryToBuild() PathSet lockFiles; /* FIXME: Should lock something like the drv itself so we don't build same CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + if (dynamic_cast(&worker.store)) + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -680,6 +688,12 @@ void DerivationGoal::tryLocalBuild() { /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ + if (!dynamic_cast(&worker.store)) { + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + } unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh new file mode 100644 index 000000000..ed51164a4 --- /dev/null +++ b/tests/binary-cache-build-remote.sh @@ -0,0 +1,13 @@ +source common.sh + +clearStore +clearCacheCache + +# Fails without remote builders +(! nix-build --store "file://$cacheDir" dependencies.nix) + +# Succeeds with default store as build remote. +nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix + +# Succeeds without any build capability because no-op +nix-build --store "file://$cacheDir" -j0 dependencies.nix diff --git a/tests/local.mk b/tests/local.mk index ce94ec80e..aa8b4f9bf 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -9,7 +9,9 @@ nix_tests = \ local-store.sh remote-store.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ - binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ + binary-cache.sh \ + binary-cache-build-remote.sh \ + nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ From 146af4ee9bb03968a7322a1ac70dc60c8d5a35e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 16:43:09 +0100 Subject: [PATCH 024/725] Move sodium_init() call --- src/libmain/shared.cc | 9 +++++++++ src/nix-store/nix-store.cc | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2247aeca4..e9f067e35 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,6 +18,10 @@ #include +#if HAVE_SODIUM +#include +#endif + namespace nix { @@ -126,6 +130,11 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif +#if HAVE_SODIUM + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); +#endif + loadConfFile(); startSignalHandlerThread(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54394e921..e1ccece99 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -981,9 +981,6 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string publicKeyFile = *i++; #if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:04:46 +0100 Subject: [PATCH 025/725] crypto.cc: API cleanup and add generate() / to_string() methods --- src/libstore/crypto.cc | 33 ++++++++++++++++++++++++++------- src/libstore/crypto.hh | 24 ++++++++++++++++-------- src/nix-store/nix-store.cc | 17 +++-------------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..135ced277 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -8,15 +8,15 @@ namespace nix { -static std::pair split(const string & s) +static std::pair split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,7 +29,12 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) +std::string Key::to_string() const +{ + return name + ":" + base64Encode(key); +} + +SecretKey::SecretKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s) } #endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { #if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; @@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const #endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig, #if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘:’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e1ccece99..e43788bc3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -19,10 +19,6 @@ #include #include -#if HAVE_SODIUM -#include -#endif - namespace nix_store { @@ -980,18 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string secretKeyFile = *i++; string publicKeyFile = *i++; -#if HAVE_SODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); + auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif + writeFile(secretKeyFile, secretKey.to_string()); } From 9374c2baeabe45a22e4b8746dc97f5ce4f030184 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:41:16 +0100 Subject: [PATCH 026/725] Add commands for generating secret/public keys --- src/nix/hash.cc | 5 -- src/nix/key-convert-secret-to-public.md | 19 ++++++ src/nix/key-generate-secret.md | 48 ++++++++++++++ src/nix/sigs.cc | 87 +++++++++++++++++++++++++ tests/binary-cache.sh | 12 ++-- 5 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 src/nix/key-convert-secret-to-public.md create mode 100644 src/nix/key-generate-secret.md diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 101b67e6a..6fd791f41 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,11 +132,6 @@ struct CmdHash : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..6ff1e1c9b --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign-paths --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 37b8a6712..b2e598ad5 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -141,3 +141,90 @@ struct CmdSignPaths : StorePathsCommand }; static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); + +#if HAVE_SODIUM +struct CmdKeyGenerateSecret : Command +{ + std::optional keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "identifier of the key (e.g. `cache.example.org-1`)", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref(); }}, + {"convert-secret-to-public", []() { return make_ref(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand("key"); +#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 92ed36225..1a06404ed 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -131,14 +131,14 @@ if [ -n "$HAVE_SODIUM" ]; then clearCache clearCacheCache -declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) -publicKey="$(cat $TEST_ROOT/pk1)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 +publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) -res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2)) -badKey="$(cat $TEST_ROOT/pk2)" +nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 +badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) -res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) -otherKey="$(cat $TEST_ROOT/pk3)" +nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 +otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) _NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:56:53 +0100 Subject: [PATCH 027/725] Make sodium a required dependency --- Makefile.config.in | 1 - configure.ac | 6 +----- perl/Makefile.config.in | 1 - perl/configure.ac | 6 +----- perl/lib/Nix/Store.xs | 10 ---------- src/libmain/shared.cc | 6 ------ src/libstore/crypto.cc | 29 ----------------------------- src/nix/sigs.cc | 2 -- tests/binary-cache.sh | 4 ---- tests/common.sh.in | 1 - 10 files changed, 2 insertions(+), 64 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 3845b3be0..d1e59e4e7 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,7 +10,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -HAVE_SODIUM = @HAVE_SODIUM@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ diff --git a/configure.ac b/configure.ac index c1bfc9b53..2047ed8d2 100644 --- a/configure.ac +++ b/configure.ac @@ -203,11 +203,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA ]) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index c87d4817e..eccfbd9f6 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -2,7 +2,6 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -HAVE_SODIUM = @HAVE_SODIUM@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ diff --git a/perl/configure.ac b/perl/configure.ac index 255744afd..85183c005 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -40,11 +40,7 @@ AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) # Look for libsodium, an optional dependency. -PKG_CHECK_MODULES([SODIUM], [libsodium], - [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) - CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" - have_sodium=1], [have_sodium=]) -AC_SUBST(HAVE_SODIUM, [$have_sodium]) +PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). perlFlags="-I$perllibdir" diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e3b7d389..ad9042a2a 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -14,9 +14,7 @@ #include "util.hh" #include "crypto.hh" -#if HAVE_SODIUM #include -#endif using namespace nix; @@ -239,12 +237,8 @@ SV * convertHash(char * algo, char * s, int toBase32) SV * signString(char * secretKey_, char * msg) PPCODE: try { -#if HAVE_SODIUM auto sig = SecretKey(secretKey_).signDetached(msg); XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } @@ -253,7 +247,6 @@ SV * signString(char * secretKey_, char * msg) int checkSignature(SV * publicKey_, SV * sig_, char * msg) CODE: try { -#if HAVE_SODIUM STRLEN publicKeyLen; unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) @@ -265,9 +258,6 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) throw Error("signature is not valid"); RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e9f067e35..6751a3744 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,9 +18,7 @@ #include -#if HAVE_SODIUM #include -#endif namespace nix { @@ -130,10 +128,8 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif -#if HAVE_SODIUM if (sodium_init() == -1) throw Error("could not initialise libsodium"); -#endif loadConfFile(); @@ -283,9 +279,7 @@ void printVersion(const string & programName) #if HAVE_BOEHMGC cfg.push_back("gc"); #endif -#if HAVE_SODIUM cfg.push_back("signed-caches"); -#endif std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 135ced277..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,9 +2,7 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include -#endif namespace nix { @@ -37,70 +35,46 @@ std::string Key::to_string() const SecretKey::SecretKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); -#endif } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -} -#endif - std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } SecretKey SecretKey::generate(std::string_view name) { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); -#else - noSodium(); -#endif } PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); auto key = publicKeys.find(std::string(ss.first)); @@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index b2e598ad5..14e2c9761 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -142,7 +142,6 @@ struct CmdSignPaths : StorePathsCommand static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); -#if HAVE_SODIUM struct CmdKeyGenerateSecret : Command { std::optional keyName; @@ -227,4 +226,3 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); -#endif diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 1a06404ed..355a37d97 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -125,8 +125,6 @@ grep -q "copying path.*input-0" $TEST_ROOT/log grep -q "copying path.*top" $TEST_ROOT/log -if [ -n "$HAVE_SODIUM" ]; then - # Create a signed binary cache. clearCache clearCacheCache @@ -181,8 +179,6 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" -fi # HAVE_LIBSODIUM - unset _NIX_FORCE_HTTP diff --git a/tests/common.sh.in b/tests/common.sh.in index 5e00d64f1..5489c0c44 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -34,7 +34,6 @@ coreutils=@coreutils@ export dot=@dot@ export SHELL="@bash@" export PAGER=cat -export HAVE_SODIUM="@HAVE_SODIUM@" export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: [PATCH 028/725] Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 232 --------------- src/nix/local.mk | 1 - src/nix/prefetch.cc | 352 +++++++++++++++++++++++ src/nix/store-prefetch-file.md | 32 +++ src/nix/store-prefetch-tarball.md | 31 ++ 6 files changed, 416 insertions(+), 234 deletions(-) delete mode 100644 src/nix-prefetch-url/nix-prefetch-url.cc create mode 100644 src/nix/prefetch.cc create mode 100644 src/nix/store-prefetch-file.md create mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include - -#include -#include -#include - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional expectedHash; - Hash hash(ht); - std::optional storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md new file mode 100644 index 000000000..535d7e022 --- /dev/null +++ b/src/nix/store-prefetch-tarball.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download a tarball and unpack it, unless it already exists in the + Nix store: + + ```console + # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ + --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= + ``` + +# Description + +This command downloads a tarball or zip file from *url*, unpacks it, +and adds the unpacked tree to the Nix store. It prints out the +resulting store path and the NAR hash of that store path. + +The name component of the store path defaults to `source`, but this +can be overriden using `--name`. + +)"" From 93f1678ec60bcacfcc857f361b5f63e37c498eb4 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 01:53:57 +0000 Subject: [PATCH 029/725] Allow Flake inputs to accept boolean and integer attributes I believe that this makes it possible to do things like Git inputs with submodules, but it also likely applies to other input types from libfetchers. --- src/libexpr/flake/flake.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4f021570c..41c93bcaa 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,11 +120,16 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) + if (attr.value->type() == nString) { attrs.emplace(attr.name, attr.value->string.s); - else - throw TypeError("flake input attribute '%s' is %s while a string is expected", + } else if (attr.value->type() == nBool) { + attrs.emplace(attr.name, Explicit{ attr.value->boolean }); + } else if (attr.value->type() == nInt) { + attrs.emplace(attr.name, attr.value->integer); + } else { + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", attr.name, showType(*attr.value)); + } } } catch (Error & e) { e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); From ba0f841a078402f95cf93693c3749743c3ab6246 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Fri, 8 Jan 2021 03:13:42 +0000 Subject: [PATCH 030/725] Use switch statement instead of sequence of ifs --- src/libexpr/flake/flake.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 41c93bcaa..9f1e4063f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -120,15 +120,19 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type() == nString) { - attrs.emplace(attr.name, attr.value->string.s); - } else if (attr.value->type() == nBool) { - attrs.emplace(attr.name, Explicit{ attr.value->boolean }); - } else if (attr.value->type() == nInt) { - attrs.emplace(attr.name, attr.value->integer); - } else { - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", - attr.name, showType(*attr.value)); + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + attr.name, showType(*attr.value)); } } } catch (Error & e) { From 48a9be2aabf6620ceb00caf7c4c917e4e0a81446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 10:44:55 +0100 Subject: [PATCH 031/725] Remove mkIntFlag --- src/libutil/args.hh | 26 +++++++++++++++----------- src/nix/verify.cc | 10 +++++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6ed541a32..3e84ac64a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -68,8 +68,12 @@ protected: , arity(ArityAny) { } - template - Handler(T * dest) + Handler(std::string * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) : fun([=](std::vector ss) { *dest = ss[0]; }) , arity(1) { } @@ -79,6 +83,15 @@ protected: : fun([=](std::vector ss) { *dest = val; }) , arity(0) { } + + template + Handler(I * dest) + : fun([=](std::vector ss) { + if (!string2Int(ss[0], *dest)) + throw UsageError("'%s' is not an integer", ss[0]); + }) + , arity(1) + { } }; /* Flags. */ @@ -161,15 +174,6 @@ public: }); } - template - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag(shortName, longName, description, [=](I n) { - *dest = n; - }); - } - template void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 16d42349f..620109aac 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,6 +20,7 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + addFlag({ .longName = "substituter", .shortName = 's', @@ -27,7 +28,14 @@ struct CmdVerify : StorePathsCommand .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "require that each path has at least N valid signatures", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override From 1d4954e73e389d319416bf29e949b4b1cbc9ebd9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:40:36 +0100 Subject: [PATCH 032/725] Remove mkFlag integer specialisation --- src/libmain/shared.cc | 16 +++++++++++++--- src/libutil/args.hh | 18 ------------------ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6751a3744..223020378 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -211,9 +211,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); + const std::string & description, const std::string & dest) + { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"n"}, + .handler = {[=](std::string s) { + unsigned int n; + if (!string2Int(s, n)) + throw UsageError("'%s' is not an integer", s); + settings.set(dest, std::to_string(n)); + }} }); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3e84ac64a..c54b0efaf 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -174,24 +174,6 @@ public: }); } - template - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {"N"}, - .handler = {[=](std::string s) { - I n; - if (!string2Int(s, n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); From 29a445840a4f01dfb1533806f8dfc28f7dc4bee9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 11:42:44 +0100 Subject: [PATCH 033/725] Remove unused mkFlag1 --- src/libutil/args.hh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c54b0efaf..62b9516d8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -143,19 +143,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {label}, - .handler = {[=](std::string s) { fun(s); }} - }); - } - void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:22:21 +0100 Subject: [PATCH 034/725] string2Int(): Return std::optional --- src/libexpr/attr-path.cc | 14 ++++++-------- src/libexpr/get-drvs.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 +++---- src/libstore/build/derivation-goal.cc | 4 +--- src/libstore/globals.cc | 8 ++++++-- src/libstore/local-store.cc | 4 +++- src/libstore/names.cc | 12 ++++++------ src/libstore/nar-info.cc | 8 ++++++-- src/libstore/profiles.cc | 11 +++++------ src/libstore/store-api.cc | 13 +++++++------ src/libutil/args.hh | 4 +++- src/libutil/config.cc | 4 +++- src/libutil/util.hh | 18 ++++++++++++------ src/nix-env/nix-env.cc | 19 +++++++++---------- src/nix/profile.cc | 5 ++--- 17 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2d37dcb7e..9dd557205 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -52,9 +52,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,7 +63,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { if (v->type() != nAttrs) throw TypeError( @@ -82,17 +80,17 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 32c115c12..1a3990ea1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -214,8 +214,8 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int(v->string.s)) + return *n; } return def; } @@ -228,8 +228,8 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float(v->string.s)) + return *n; } return def; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index bcb904c0d..d1003de57 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme if (name == "rev" || name == "narHash") input.attrs.insert_or_assign(name, value); else if (name == "revCount" || name == "lastModified") { - uint64_t n; - if (!string2Int(value, n)) + if (auto n = string2Int(value)) + input.attrs.insert_or_assign(name, *n); + else throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input.attrs.insert_or_assign(name, n); } else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 223020378..f1feeddd6 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,10 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - unsigned int n; - if (!string2Int(s, n)) + if (auto n = string2Int(s)) + settings.set(dest, std::to_string(*n)); + else throw UsageError("'%s' is not an integer", s); - settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index ffae5d796..38f627b44 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -70,10 +70,9 @@ template N getIntArg(const string & opt, s.resize(s.size() - 1); } } - N n; - if (!string2Int(s, n)) - throw UsageError("'%1%' requires an integer argument", opt); - return n * multiplier; + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%1%' requires an integer argument", opt); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index af3ab87a9..35f365795 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ad66ef8a8..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -228,8 +228,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702e7b136..c61f34275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int(c1); + auto n2 = string2Int(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..01e2fcc7b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -932,19 +932,20 @@ std::optional decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 62b9516d8..823d843aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,7 +87,9 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (!string2Int(ss[0], *dest)) + if (auto n = string2Int(ss[0])) + *dest = *n; + else throw UsageError("'%s' is not an integer", ss[0]); }) , arity(1) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7af3e7883..7467e5ac0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -230,7 +230,9 @@ template void BaseSetting::set(const std::string & str, bool append) { static_assert(std::is_integral::value, "Integer required."); - if (!string2Int(str, value)) + if (auto n = string2Int(str)) + value = *n; + else throw UsageError("setting '%s' has invalid value '%s'", name, str); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0f82bed78..7a4d5fe92 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -397,21 +397,27 @@ bool statusOk(int status); /* Parse a string into an integer. */ -template bool string2Int(const string & s, N & n) +template +std::optional string2Int(const std::string & s) { - if (string(s, 0, 1) == "-" && !std::numeric_limits::is_signed) - return false; + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return {}; std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } /* Parse a string into a float. */ -template bool string2Float(const string & s, N & n) +template +std::optional string2Float(const string & s) { std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return {}; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c2e075ed..9963f05d9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - GenerationNumber dstGen; - if (!string2Int(opArgs.front(), dstGen)) + if (auto dstGen = string2Int(opArgs.front())) + switchGeneration(globals, *dstGen); + else throw UsageError("expected a generation number"); - - switchGeneration(globals, dstGen); } @@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - GenerationNumber max; - if (!string2Int(str_max, max) || max == 0) + auto max = string2Int(str_max); + if (!max || *max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { - GenerationNumber n; - if (!string2Int(i, n)) + if (auto n = string2Int(i)) + gens.insert(*n); + else throw UsageError("invalid generation number '%1%'", i); - gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d8d2b3a70..8cdd34a20 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,9 +209,8 @@ public: std::vector res; for (auto & s : _matchers) { - size_t n; - if (string2Int(s, n)) - res.push_back(n); + if (auto n = string2Int(s)) + res.push_back(*n); else if (store->isStorePath(s)) res.push_back(s); else From 17beae299d5e6bb511c453d0b9d0d7ef906b3d14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:51:19 +0100 Subject: [PATCH 035/725] Support binary unit prefixes in command line arguments --- src/libmain/shared.cc | 6 ++---- src/libmain/shared.hh | 17 +---------------- src/libutil/args.hh | 5 +---- src/libutil/util.hh | 28 +++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f1feeddd6..e797c2fb9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -219,10 +219,8 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description = description, .labels = {"n"}, .handler = {[=](std::string s) { - if (auto n = string2Int(s)) - settings.set(dest, std::to_string(*n)); - else - throw UsageError("'%s' is not an integer", s); + auto n = string2IntWithUnitPrefix(s); + settings.set(dest, std::to_string(n)); }} }); }; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 38f627b44..edc7b5efa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -57,22 +57,7 @@ template N getIntArg(const string & opt, { ++i; if (i == end) throw UsageError("'%1%' requires an argument", opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%1%' requires an integer argument", opt); + return string2IntWithUnitPrefix(*i); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 823d843aa..3783bc84f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -87,10 +87,7 @@ protected: template Handler(I * dest) : fun([=](std::vector ss) { - if (auto n = string2Int(ss[0])) - *dest = *n; - else - throw UsageError("'%s' is not an integer", ss[0]); + *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7a4d5fe92..ab0bd865a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,12 +401,34 @@ template std::optional string2Int(const std::string & s) { if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return {}; + return std::nullopt; std::istringstream str(s); N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; +} + +/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + 'T' denoting a binary unit prefix. */ +template +N string2IntWithUnitPrefix(std::string s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.resize(s.size() - 1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); } /* Parse a string into a float. */ @@ -417,7 +439,7 @@ std::optional string2Float(const string & s) N n; str >> n; if (str && str.get() == EOF) return n; - return {}; + return std::nullopt; } From e21aee58f6dd7785df50d5d2a473feb5f6b2ed4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 14:17:06 +0100 Subject: [PATCH 036/725] Fix tests --- src/libutil/tests/tests.cc | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 35a5d27bb..58df9c5ac 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -320,20 +320,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Float, emptyString) { - double n; - ASSERT_EQ(string2Float("", n), false); + ASSERT_EQ(string2Float(""), std::nullopt); } TEST(string2Float, trivialConversions) { - double n; - ASSERT_EQ(string2Float("1.0", n), true); - ASSERT_EQ(n, 1.0); + ASSERT_EQ(string2Float("1.0"), 1.0); - ASSERT_EQ(string2Float("0.0", n), true); - ASSERT_EQ(n, 0.0); + ASSERT_EQ(string2Float("0.0"), 0.0); - ASSERT_EQ(string2Float("-100.25", n), true); - ASSERT_EQ(n, (-100.25)); + ASSERT_EQ(string2Float("-100.25"), -100.25); } /* ---------------------------------------------------------------------------- @@ -341,20 +336,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Int, emptyString) { - double n; - ASSERT_EQ(string2Int("", n), false); + ASSERT_EQ(string2Int(""), std::nullopt); } TEST(string2Int, trivialConversions) { - double n; - ASSERT_EQ(string2Int("1", n), true); - ASSERT_EQ(n, 1); + ASSERT_EQ(string2Int("1"), 1); - ASSERT_EQ(string2Int("0", n), true); - ASSERT_EQ(n, 0); + ASSERT_EQ(string2Int("0"), 0); - ASSERT_EQ(string2Int("-100", n), true); - ASSERT_EQ(n, (-100)); + ASSERT_EQ(string2Int("-100"), -100); } /* ---------------------------------------------------------------------------- From 1db3f84baccc30ac38227c1f7edc3bfbc8e5ff5b Mon Sep 17 00:00:00 2001 From: Danila Date: Fri, 8 Jan 2021 16:12:21 -0800 Subject: [PATCH 037/725] Upcase "Boolean" in Flake attribute type error Co-authored-by: Eelco Dolstra --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9f1e4063f..61aeae543 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, boolean, or integer is expected", + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", attr.name, showType(*attr.value)); } } From fdcd62eec59485665b919c048874de05235b5971 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Jan 2021 23:20:02 +0100 Subject: [PATCH 038/725] Add 'nix store gc' command --- src/nix/store-gc.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ src/nix/store-gc.md | 21 +++++++++++++++++++++ tests/flakes.sh | 6 +++--- 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/nix/store-gc.cc create mode 100644 src/nix/store-gc.md diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..6e9607d03 --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "stop after freeing `n` bytes of disk space", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/tests/flakes.sh b/tests/flakes.sh index 5aec563ac..2b7bcdd68 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -276,18 +276,18 @@ git -C $flake3Dir commit -m 'Add lockfile' # Test whether registry caching works. nix registry list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp -nix-store --gc +nix store gc nix registry list --flake-registry file://$registry --refresh | grep -q flake3 mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache -nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache +nix store gc # get rid of copies in the store to ensure they get fetched to our git cache _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar mv $flake1Dir $flake1Dir.tmp mv $flake2Dir $flake2Dir.tmp -nix-store --gc +nix store gc _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar _NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh mv $flake1Dir.tmp $flake1Dir From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 12:36:39 +0100 Subject: [PATCH 039/725] nix store prefetch-tarball -> nix flake prefetch --- src/nix/flake-prefetch.md | 28 +++++++++++ src/nix/flake.cc | 40 ++++++++++++++++ src/nix/prefetch.cc | 77 +++++++++---------------------- src/nix/store-prefetch-tarball.md | 31 ------------- 4 files changed, 89 insertions(+), 87 deletions(-) create mode 100644 src/nix/flake-prefetch.md delete mode 100644 src/nix/store-prefetch-tarball.md diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2b91faa64..b73b9cf4e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -960,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand } }; +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + struct CmdFlake : NixMultiCommand { CmdFlake() @@ -973,6 +1012,7 @@ struct CmdFlake : NixMultiCommand {"clone", []() { return make_ref(); }}, {"archive", []() { return make_ref(); }}, {"show", []() { return make_ref(); }}, + {"prefetch", []() { return make_ref(); }}, }) { } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 969299489..ce8c85ecf 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv) static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); -struct CmdStorePrefetch : StoreCommand, MixJSON +struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; - bool unpack; std::optional name; HashType hashType = htSHA256; std::optional expectedHash; - CmdStorePrefetch(bool unpack) - : unpack(unpack) + CmdStorePrefetchFile() { addFlag({ .longName = "name", @@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag({ .longName = "expected-hash", - .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .description = "expected hash of the file", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,14 +274,31 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + expectArg("url", &url); } Category category() override { return catUtility; } + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); @@ -299,54 +314,4 @@ struct CmdStorePrefetch : StoreCommand, MixJSON } }; -struct CmdStorePrefetchFile : CmdStorePrefetch -{ - CmdStorePrefetchFile() - : CmdStorePrefetch(false) - { - name = "source"; - - addFlag({ - .longName = "executable", - .description = "make the resulting file executable", - .handler = {&executable, true}, - }); - } - - std::string description() override - { - return "download a file into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-file.md" - ; - } -}; - static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); - -struct CmdStorePrefetchTarball : CmdStorePrefetch -{ - CmdStorePrefetchTarball() - : CmdStorePrefetch(true) - { - name = "source"; - } - - std::string description() override - { - return "download and unpack a tarball into the Nix store"; - } - - std::string doc() override - { - return - #include "store-prefetch-tarball.md" - ; - } -}; - -static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); diff --git a/src/nix/store-prefetch-tarball.md b/src/nix/store-prefetch-tarball.md deleted file mode 100644 index 535d7e022..000000000 --- a/src/nix/store-prefetch-tarball.md +++ /dev/null @@ -1,31 +0,0 @@ -R""( - -# Examples - -* Download a tarball and unpack it: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz - Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz' - to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash - 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). - ``` - -* Download a tarball and unpack it, unless it already exists in the - Nix store: - - ```console - # nix store prefetch-tarball https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz \ - --expected-hash sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY= - ``` - -# Description - -This command downloads a tarball or zip file from *url*, unpacks it, -and adds the unpacked tree to the Nix store. It prints out the -resulting store path and the NAR hash of that store path. - -The name component of the store path defaults to `source`, but this -can be overriden using `--name`. - -)"" From 77c9ceda4be8dd304b07f654d8c059a01d839108 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:42:24 +0100 Subject: [PATCH 040/725] Tweak --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 827adbd05..361c20cc9 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -226,7 +226,7 @@ control what gets deleted and in what order: or TiB units. The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` variables in the Nix +`keep-outputs` and `keep-derivations` settings in the Nix configuration file. By default, the collector prints the total number of freed bytes when it From 6254b1f5d298ff73127d7b0f0da48f142bdc753c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 19:46:17 +0100 Subject: [PATCH 041/725] Add 'nix store delete' command --- src/nix/store-delete.cc | 45 +++++++++++++++++++++++++++++++++++++++ src/nix/store-delete.md | 24 +++++++++++++++++++++ tests/multiple-outputs.sh | 2 +- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/nix/store-delete.cc create mode 100644 src/nix/store-delete.md diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..f3677763c --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,45 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "do not check whether the paths are reachable from a root", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 7a6ec181d..de573d4fa 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou # Delete one of the outputs and rebuild it. This will cause a hash # rewrite. -nix-store --delete $TEST_ROOT/result-second --ignore-liveness +nix store delete $TEST_ROOT/result-second --ignore-liveness nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result [ "$(cat $TEST_ROOT/result-second/file)" = "second" ] [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Tue, 12 Jan 2021 01:28:00 +0100 Subject: [PATCH 042/725] Don't let 'preferLocalBuild' override 'max-jobs=0' This resolves #3810 by changing the behavior of `max-jobs = 0`, so that specifying the option also avoids local building of derivations with the attribute `preferLocalBuild = true`. --- src/libstore/parsed-derivations.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 11 Jan 2021 22:05:32 -0600 Subject: [PATCH 043/725] Set kern.curproc_arch_affinity=0 to escape Rosetta By default, once you enter x86_64 Rosetta 2, macOS will try to run everything in x86_64. So an x86_64 Nix will still try to use x86_64 even when system = aarch64-darwin. To avoid this we can set kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0, we ignore this preference. This is based on how https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html works. Completely undocumented, but seems to work! Note, you can verify this works with this impure Nix expression: ``` { a = derivation { name = "a"; system = "aarch64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = arm64 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; b = derivation { name = "b"; system = "x86_64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; } ``` --- src/libstore/build/derivation-goal.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..a02ddb950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,6 +52,7 @@ #if __APPLE__ #include +#include #endif #include @@ -2869,6 +2870,10 @@ void DerivationGoal::runChild() throw SysError("failed to initialize builder"); if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } else if (drv->platform == "x86_64-darwin") { From 29007f8bc6ea42ae1f8311f00c0b5e14f04ec9e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 19:57:05 +0100 Subject: [PATCH 044/725] nix profile info -> nix profile list --- src/nix/{profile-info.md => profile-list.md} | 2 +- src/nix/profile.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/nix/{profile-info.md => profile-list.md} (98%) diff --git a/src/nix/profile-info.md b/src/nix/profile-list.md similarity index 98% rename from src/nix/profile-info.md rename to src/nix/profile-list.md index a0c04fc8c..5c29c0b02 100644 --- a/src/nix/profile-info.md +++ b/src/nix/profile-list.md @@ -5,7 +5,7 @@ R""( * Show what packages are installed in the default profile: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cdd34a20..ac60d336c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { @@ -346,7 +346,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro std::string doc() override { return - #include "profile-info.md" + #include "profile-list.md" ; } @@ -408,7 +408,7 @@ struct CmdProfile : NixMultiCommand {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, + {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, }) { } From 2f463e90ed077e066455a9ef6e024b18fd61c4de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Jan 2021 23:51:07 +0100 Subject: [PATCH 045/725] Add 'nix profile history' command Replaces 'nix-env --list-generations'. Similar to 'nix profile diff-closures' but shows only the changes in top-level packages. --- src/nix/command.hh | 2 + src/nix/profile-history.md | 26 +++++++++ src/nix/profile.cc | 114 +++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/nix/profile-history.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 6882db195..3aae57edd 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -261,6 +261,8 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); +std::string showVersions(const std::set & versions); + void printClosureDiff( ref store, const StorePath & beforePath, diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..d0fe40c82 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 -> 509: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 -> 510: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ac60d336c..ca95817d0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include "../nix-env/user-env.hh" #include "profiles.hh" +#include "names.hh" #include #include @@ -21,6 +22,13 @@ struct ProfileElementSource FlakeRef resolvedRef; std::string attrPath; // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } }; struct ProfileElement @@ -29,6 +37,29 @@ struct ProfileElement std::optional source; bool active = true; // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } }; struct ProfileManifest @@ -142,6 +173,46 @@ struct ProfileManifest return std::move(info.path); } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } }; struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile @@ -401,6 +472,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile } }; +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + if (prevGen) + std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); + else + std::cout << fmt("Version %d:\n", gen.number); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -410,6 +523,7 @@ struct CmdProfile : NixMultiCommand {"upgrade", []() { return make_ref(); }}, {"list", []() { return make_ref(); }}, {"diff-closures", []() { return make_ref(); }}, + {"history", []() { return make_ref(); }}, }) { } From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 13 Jan 2021 10:13:51 +0100 Subject: [PATCH 046/725] Remove a redundant condition in DerivationGoal::tryLocalBuild() --- src/libstore/build/derivation-goal.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..415a55d37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 14:18:04 +0100 Subject: [PATCH 047/725] Convert option descriptions to Markdown --- src/libexpr/common-eval-args.cc | 10 +++++----- src/libmain/common-args.cc | 13 ++++++------- src/libmain/common-args.hh | 4 ++-- src/libmain/shared.cc | 20 ++++++++++---------- src/nix/add-to-store.cc | 2 +- src/nix/build.cc | 6 +++--- src/nix/bundle.cc | 6 +++--- src/nix/command.cc | 14 +++++++------- src/nix/copy.cc | 8 ++++---- src/nix/develop.cc | 16 ++++++++-------- src/nix/eval.cc | 6 +++--- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 19 +++++++++---------- src/nix/installables.cc | 26 +++++++++++++------------- src/nix/ls.cc | 6 +++--- src/nix/main.cc | 14 +++++++------- src/nix/path-info.cc | 8 ++++---- src/nix/prefetch.cc | 8 +++++--- src/nix/run.cc | 2 +- src/nix/show-derivation.cc | 2 +- src/nix/sigs.cc | 6 +++--- src/nix/store-delete.cc | 2 +- src/nix/store-gc.cc | 2 +- src/nix/upgrade-nix.cc | 4 ++-- src/nix/verify.cc | 8 ++++---- src/nix/why-depends.cc | 2 +- 26 files changed, 111 insertions(+), 111 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..ffe782454 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs() { addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up `<...>` file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); addFlag({ .longName = "impure", - .description = "allow access to mutable paths and repositories", + .description = "Allow access to mutable paths and repositories.", .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", - .description = "override a flake registry value", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3e4e475e5..bd5573e5d 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "verbose", .shortName = 'v', - .description = "increase verbosity level", + .description = "Increase the logging verbosity level.", .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", - .description = "decrease verbosity level", + .description = "Decrease the logging verbosity level.", .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", - .description = "enable debug output", + .description = "Set the logging verbosity level to 'debug'.", .handler = {[]() { verbosity = lvlDebug; }}, }); addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding `nix.conf`)", + .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { @@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "format of log output; `raw`, `internal-json`, `bar` " - "or `bar-with-logs`", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "max-jobs", .shortName = 'j', - .description = "maximum number of parallel builds", + .description = "The maximum number of parallel builds.", .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index a4de3dccf..47f341619 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -16,7 +16,7 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); } }; @@ -26,7 +26,7 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "produce JSON output", &json); + mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); } }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e797c2fb9..7e27e95c2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -186,27 +186,27 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-build-output", .shortName = 'Q', - .description = "do not show build output", + .description = "Do not show build output.", .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ .longName = "keep-failed", .shortName ='K', - .description = "keep temporary directories of failed builds", + .description = "Keep temporary directories of failed builds.", .handler = {&(bool&) settings.keepFailed, true}, }); addFlag({ .longName = "keep-going", .shortName ='k', - .description = "keep going after a build fails", + .description = "Keep going after a build fails.", .handler = {&(bool&) settings.keepGoing, true}, }); addFlag({ .longName = "fallback", - .description = "build from source if substitution fails", + .description = "Build from source if substitution fails.", .handler = {&(bool&) settings.tryFallback, true}, }); @@ -225,19 +225,19 @@ LegacyArgs::LegacyArgs(const std::string & programName, }); }; - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores"); + intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); + intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "do not write to the Nix store", + mkFlag(0, "readonly-mode", "Do not write to the Nix store.", &settings.readOnlyMode); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", &gcWarning, false); addFlag({ .longName = "store", - .description = "URI of the Nix store to use", + .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, .handler = {&(std::string&) settings.storeUri}, }); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ea4bbbab9..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); diff --git a/src/nix/build.cc b/src/nix/build.cc index c2974d983..4cb8ade08 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); addFlag({ .longName = "rebuild", - .description = "rebuild an already built package and compare the result to the existing store paths", + .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, }); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5f558b01e..1789e4598 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand { addFlag({ .longName = "bundler", - .description = "use custom bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { @@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -90,7 +90,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/command.cc b/src/nix/command.cc index 596217775..ba58c7d6b 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) if (recursive) addFlag({ .longName = "no-recursive", - .description = "apply operation to specified paths only", + .description = "Apply operation to specified paths only.", .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', - .description = "apply operation to closure of the specified paths", + .description = "Apply operation to closure of the specified paths.", .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "apply operation to the entire store", &all); + mkFlag(0, "all", "Apply the operation to every store path.", &all); } void StorePathsCommand::run(ref store) @@ -133,7 +133,7 @@ MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "profile to update", + .description = "The profile to update.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "ignore-environment", .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", + .description = "Clear the entire environment (except those specified with `--keep`).", .handler = {&ignoreEnvironment, true}, }); addFlag({ .longName = "keep", .shortName = 'k', - .description = "keep specified environment variable", + .description = "Keep the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { keep.insert(s); }}, }); @@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "unset", .shortName = 'u', - .description = "unset specified environment variable", + .description = "Unset the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { unset.insert(s); }}, }); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2394eb46d..f15031a45 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index edd87f246..578258394 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile { addFlag({ .longName = "redirect", - .description = "redirect a store path to a mutable location", + .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); @@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed instead of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "phase", - .description = "phase to run (e.g. `build` or `configure`)", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", .labels = {"phase-name"}, .handler = {&phase}, }); addFlag({ .longName = "configure", - .description = "run the configure phase", + .description = "Run the `configure` phase.", .handler = {&phase, {"configure"}}, }); addFlag({ .longName = "build", - .description = "run the build phase", + .description = "Run the `build` phase.", .handler = {&phase, {"build"}}, }); addFlag({ .longName = "check", - .description = "run the check phase", + .description = "Run the `check` phase.", .handler = {&phase, {"check"}}, }); addFlag({ .longName = "install", - .description = "run the install phase", + .description = "Run the `install` phase.", .handler = {&phase, {"install"}}, }); addFlag({ .longName = "installcheck", - .description = "run the installcheck phase", + .description = "Run the `installcheck` phase.", .handler = {&phase, {"installCheck"}}, }); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 321df7495..b5049ac65 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); addFlag({ .longName = "apply", - .description = "apply a function to each argument", + .description = "Apply the function *expr* to each argument.", .labels = {"expr"}, .handler = {&apply}, }); addFlag({ .longName = "write-to", - .description = "write a string or attrset of strings to 'path'", + .description = "Write a string or attrset of strings to *path*.", .labels = {"path"}, .handler = {&writeTo}, }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b73b9cf4e..4cd7d77a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -222,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand { addFlag({ .longName = "no-build", - .description = "do not build checks", + .description = "Do not build checks.", .handler = {&build, false} }); } @@ -573,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand addFlag({ .longName = "template", .shortName = 't', - .description = "the template to use", + .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, .completer = {[&](size_t, std::string_view prefix) { @@ -717,7 +717,7 @@ struct CmdFlakeClone : FlakeCommand addFlag({ .longName = "dest", .shortName = 'f', - .description = "destination path", + .description = "Clone the flake to path *dest*.", .labels = {"path"}, .handler = {&destDir} }); @@ -807,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand { addFlag({ .longName = "legacy", - .description = "show the contents of the 'legacyPackages' output", + .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6fd791f41..79d506ace 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,15 +19,15 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); + mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); + mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); + mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") - .description("compute hash modulo specified string") + .description("Compute the hash modulo specified the string.") .labels({"modulus"}) .dest(&modulus); #endif @@ -40,15 +40,14 @@ struct CmdHashBase : Command std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; - break; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } void run() override diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3506c3fcc..50e3b29c4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions() { addFlag({ .longName = "recreate-lock-file", - .description = "recreate lock file from scratch", + .description = "Recreate the flake's lock file from scratch.", .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", - .description = "do not allow any updates to the lock file", + .description = "Do not allow any updates to the flake's lock file.", .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", - .description = "do not write the newly generated lock file", + .description = "Do not write the flake's newly generated lock file.", .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", - .description = "don't use flake registries", + .description = "Don't allow lookups in the flake registries.", .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", - .description = "commit changes to the lock file", + .description = "Commit changes to the flake's lock file.", .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", - .description = "update a specific flake input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", - .description = "use the inputs of the specified flake as registry entries", + .description = "Use the inputs of the specified flake as registry entries.", .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "evaluate *file* rather than the default", + .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", .labels = {"file"}, .handler = {&file}, .completer = completePath }); addFlag({ - .longName ="expr", - .description = "evaluate attributes from *expr*", + .longName = "expr", + .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", .labels = {"expr"}, .handler = {&expr} }); addFlag({ - .longName ="derivation", - .description = "operate on the store derivation rather than its outputs", + .longName = "derivation", + .description = "Operate on the store derivation rather than its outputs.", .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d48287f27..c0b1ecb32 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); + mkFlag('l', "long", "Show detailed file information.", &verbose); + mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); } void listText(ref accessor) diff --git a/src/nix/main.cc b/src/nix/main.cc index b2406fafe..803453dd5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -69,15 +69,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", + .description = "Show usage information.", .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ .longName = "help-config", - .description = "show configuration options", + .description = "Show configuration settings.", .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; + std::cout << "The following configuration settings are available:\n\n"; Table2 tbl; std::map settings; globalConfig.getSettings(settings); @@ -91,25 +91,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", + .description = "Show version information.", .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 30b6a50f8..0fa88f1bf 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); + mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); + mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); + mkFlag(0, "sigs", "Show signatures.", &showSigs); } std::string description() override diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce8c85ecf..a831dcd15 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { addFlag({ .longName = "name", - .description = "store path name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, .handler = {&name} }); addFlag({ .longName = "expected-hash", - .description = "expected hash of the file", + .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON addFlag({ .longName = "executable", - .description = "make the resulting file executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", .handler = {&executable, true}, }); diff --git a/src/nix/run.cc b/src/nix/run.cc index 1340dd46f..ec9388234 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 13f2c8e69..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 14e2c9761..4b6ead6c7 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -101,7 +101,7 @@ struct CmdSignPaths : StorePathsCommand addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath @@ -150,7 +150,7 @@ struct CmdKeyGenerateSecret : Command { addFlag({ .longName = "key-name", - .description = "identifier of the key (e.g. `cache.example.org-1`)", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", .labels = {"name"}, .handler = {&keyName}, }); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index f3677763c..9c8fef191 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -13,7 +13,7 @@ struct CmdStoreDelete : StorePathsCommand { addFlag({ .longName = "ignore-liveness", - .description = "do not check whether the paths are reachable from a root", + .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true} }); } diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 6e9607d03..a2d74066e 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -13,7 +13,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun { addFlag({ .longName = "max", - .description = "stop after freeing `n` bytes of disk space", + .description = "Stop after freeing *n* bytes of disk space.", .labels = {"n"}, .handler = {&options.maxFreed} }); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 79be31e73..299ea40aa 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 620109aac..b2963cf74 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,13 +18,13 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); + mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); @@ -32,7 +32,7 @@ struct CmdVerify : StorePathsCommand addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "require that each path has at least N valid signatures", + .description = "Require that each path has at least *n* valid signatures.", .labels = {"n"}, .handler = {&sigsNeeded} }); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 297b638cc..7a4ca5172 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:27:39 +0100 Subject: [PATCH 048/725] Add 'nix store repair' command --- src/libstore/local-store.hh | 4 +--- src/libstore/store-api.hh | 5 +++++ src/nix-store/nix-store.cc | 2 +- src/nix/store-delete.cc | 1 - src/nix/store-repair.cc | 27 +++++++++++++++++++++++++++ src/nix/store-repair.md | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/nix/store-repair.cc create mode 100644 src/nix/store-repair.md diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d29c5960..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..d1b83933a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -604,6 +604,11 @@ public: virtual ref getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e43788bc3..b97f684a4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -757,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); + store->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 9c8fef191..10245978e 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,6 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector storePaths) override { - for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref store, std::vector storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" From d33eca8539d2e66759f7b52fa7b0db4a6a1ba673 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:31:18 +0100 Subject: [PATCH 049/725] Rename 'nix store sign-paths' to 'nix store sign' --- doc/manual/src/advanced-topics/post-build-hook.md | 4 ++-- src/nix/key-generate-secret.md | 2 +- src/nix/main.cc | 2 +- src/nix/sigs.cc | 6 +++--- tests/signing.sh | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index bbdabed41..fcb52d878 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -53,7 +53,7 @@ set -f # disable globbing export IFS=' ' echo "Signing paths" $OUT_PATHS -nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS exec nix copy --to 's3://example-nix-cache' $OUT_PATHS ``` @@ -63,7 +63,7 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix -> sign-paths`. Nix guarantees the paths will not contain any spaces, +> store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md index 6ff1e1c9b..4938f637c 100644 --- a/src/nix/key-generate-secret.md +++ b/src/nix/key-generate-secret.md @@ -12,7 +12,7 @@ R""( ```console # nix build nixpkgs#hello - # nix store sign-paths --key-file ./secret-key --recursive ./result + # nix store sign --key-file ./secret-key --recursive ./result ``` Finally, we can verify the store paths using the corresponding diff --git a/src/nix/main.cc b/src/nix/main.cc index 803453dd5..398526020 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -129,7 +129,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"make-content-addressable", {"store", "make-content-addressable"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign-paths"}}, + {"sign-paths", {"store", "sign"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 4b6ead6c7..3445182f2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -92,11 +92,11 @@ struct CmdCopySigs : StorePathsCommand static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", @@ -140,7 +140,7 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand2({"store", "sign-paths"}); +static auto rCmdSign = registerCommand2({"store", "sign"}); struct CmdKeyGenerateSecret : Command { diff --git a/tests/signing.sh b/tests/signing.sh index bd6280cc6..6aafbeb91 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -47,8 +47,8 @@ expect 2 nix store verify -r $outPath2 --sigs-needed 1 expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 -# Test "nix store sign-paths". -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPath2 +# Test "nix store sign". +nix store sign --key-file $TEST_ROOT/sk1 $outPath2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 @@ -63,7 +63,7 @@ nix store verify $outPathCA nix store verify $outPathCA --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix store sign --key-file $TEST_ROOT/sk1 $outPathCA nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 # Copy to a binary cache. @@ -76,7 +76,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) (! [[ $info =~ 'cache2.example.org' ]]) # Verify that adding a signature to a path in a binary cache works. -nix store sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] From 28ef6ebf914792f9e543e9778248f06e716a859d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:51:27 +0100 Subject: [PATCH 050/725] Typo --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index fc6195cf0..9227369b8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -318,7 +318,7 @@ static int main_nix_daemon(int argc, char * * argv) FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limitting access to thoses is explicitly + standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); } From 7a472a76d4dcbbd0eb7832c0bdcb120d32881e8b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jan 2021 00:05:04 +0100 Subject: [PATCH 051/725] Add 'nix daemon' command --- src/nix/command.hh | 2 + .../nix-daemon.cc => nix/daemon.cc} | 114 +++++++++++------- src/nix/daemon.md | 21 ++++ src/nix/main.cc | 3 + tests/common.sh.in | 2 +- 5 files changed, 98 insertions(+), 44 deletions(-) rename src/{nix-daemon/nix-daemon.cc => nix/daemon.cc} (77%) create mode 100644 src/nix/daemon.md diff --git a/src/nix/command.hh b/src/nix/command.hh index 3aae57edd..f325cd906 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -13,6 +13,8 @@ namespace nix { extern std::string programPath; +extern char * * savedArgv; + class EvalState; struct Pos; class Store; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc similarity index 77% rename from src/nix-daemon/nix-daemon.cc rename to src/nix/daemon.cc index 9227369b8..204d4ce6b 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,4 @@ +#include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" @@ -150,7 +151,7 @@ static ref openUncachedStore() } -static void daemonLoop(char * * argv) +static void daemonLoop() { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -232,9 +233,9 @@ static void daemonLoop(char * * argv) setSigChldAction(false); // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && argv[1]) { + if (peer.pidKnown && savedArgv[1]) { string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); } // Handle the connection. @@ -264,6 +265,48 @@ static void daemonLoop(char * * argv) } } +static void runDaemon(bool stdio) +{ + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + } + } else + daemonLoop(); +} static int main_nix_daemon(int argc, char * * argv) { @@ -285,49 +328,34 @@ static int main_nix_daemon(int argc, char * * argv) initPlugins(); - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } - } else { - daemonLoop(argv); - } + runDaemon(stdio); return 0; } } static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/main.cc b/src/nix/main.cc index 398526020..418396280 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -52,6 +52,7 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { @@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand("help"); void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { diff --git a/tests/common.sh.in b/tests/common.sh.in index 5489c0c44..e3bcab507 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -73,7 +73,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix-daemon & + nix daemon & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 From 86a2ceeb986609488be1c6794a8e416df3b90c7b Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Wed, 13 Jan 2021 16:56:23 -0800 Subject: [PATCH 052/725] Fix gcc10 build --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 61aeae543..0786fef3d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -128,7 +128,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(attr.name, Explicit { attr.value->boolean }); break; case nInt: - attrs.emplace(attr.name, attr.value->integer); + attrs.emplace(attr.name, (long unsigned int)attr.value->integer); break; default: throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", From 7af743470c09b835f910d2e25786c080ccfe52c1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 15 Jan 2021 16:37:41 +0000 Subject: [PATCH 053/725] Make public keys and `requireSigs` local-store specific again Thanks @regnat and @edolstra for catching this and comming up with the solution. They way I had generalized those is wrong, because local settings for non-local stores is confusing default. And due to the nature of C++ inheritance, fixing the defaults is more annoying than it should be. Additionally, I thought we might just drop the check in the substitution logic since `Store::addToStore` is now streaming, but @regnat rightfully pointed out that as it downloads dependencies first, that would still be too late, and also waste effort on possibly unneeded/unwanted dependencies. The simple and correct thing to do is just make a store method for the boolean logic, keeping all the setting and key stuff the way it was before. That new method is both used by `LocalStore::addToStore` and the substitution goal check. Perhaps we might eventually make it fancier, e.g. sending the ValidPathInfo to remote stores for them to validate, but this is good enough for now. --- src/libstore/build/substitution-goal.cc | 4 +--- src/libstore/local-store.cc | 14 ++++++++++++- src/libstore/local-store.hh | 14 +++++++++++++ src/libstore/misc.cc | 9 -------- src/libstore/store-api.hh | 28 +++++++++++++------------ 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..f3c9040bc 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -142,9 +142,7 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) + if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) { logWarning({ .name = "Invalid path signature", diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4f48522c6..d6d74a0b0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1098,11 +1098,23 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } } +const PublicKeys & LocalStore::getPublicKeys() +{ + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *state->publicKeys; +} + +bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +{ + return requireSigs && !info.checkSignatures(*this, getPublicKeys()); +} void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + if (checkSigs && pathInfoIsTrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69704d266..9d235ba0a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,6 +35,10 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + Setting requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + const std::string name() override { return "Local Store"; } }; @@ -71,6 +75,8 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); + + std::unique_ptr publicKeys; }; Sync _state; @@ -88,6 +94,12 @@ public: const Path tempRootsDir; const Path fnTempRoots; +private: + + const PublicKeys & getPublicKeys(); + +public: + // Hack for build-remote.cc. PathSet locksHeld; @@ -124,6 +136,8 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + bool pathInfoIsTrusted(const ValidPathInfo &) override; + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0d4190a56..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,13 +282,4 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } -const PublicKeys & Store::getPublicKeys() -{ - auto cryptoState(_cryptoState.lock()); - if (!cryptoState->publicKeys) - cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *cryptoState->publicKeys; -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e6a14afc3..3221cf249 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,10 +189,6 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -376,6 +372,21 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /* Check whether the given valid path info is sufficiently well-formed + (e.g. hash content-address or signature) in order to be included in the + given store. + + These same checks would be performed in addToStore, but this allows an + earlier failure in the case where dependencies need to be added too, but + the addToStore wouldn't fail until those dependencies are added. Also, + we don't really want to add the dependencies listed in a nar info we + don't trust anyyways. + */ + virtual bool pathInfoIsTrusted(const ValidPathInfo &) + { + return true; + } + protected: virtual void queryPathInfoUncached(const StorePath & path, @@ -719,20 +730,11 @@ public: return toRealPath(printStorePath(storePath)); } - const PublicKeys & getPublicKeys(); - virtual void createUser(const std::string & userName, uid_t userId) { } protected: - struct CryptoState - { - std::unique_ptr publicKeys; - }; - - Sync _cryptoState; - Stats stats; /* Unsupported methods. */ From 1e13c79a9165e99be9fccfec8e442d14bb66aef0 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sat, 16 Jan 2021 19:11:10 +0000 Subject: [PATCH 054/725] Document expected output of 'nix store ping'. While interpreting the output is fairly intuitive it would be better to explicitly specify what a good invocation looks like. That this isn't completely obvious (or at least causes folks to second-guess themselves) can be seen in a couple user threads: - https://discourse.nixos.org/t/nixos-cache-fetching-issue/3575/11 - https://discourse.nixos.org/t/newbie-question-cant-get-trivial-example-of-nixops-to-work-on-my-mac/1125/8 --- src/nix/ping-store.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 322093091..79b108d9c 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,4 +27,6 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. +When the command succeeds a zero exit code is returned with no output. + )"" From 1acbb61696c118712417bcd1c59021cc84650e16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 17 Jan 2021 19:49:28 +0100 Subject: [PATCH 055/725] Tweak --- src/nix/ping-store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md index 79b108d9c..8c846791b 100644 --- a/src/nix/ping-store.md +++ b/src/nix/ping-store.md @@ -27,6 +27,7 @@ argument `--store` *url*) can be accessed. What this means is dependent on the type of the store. For instance, for an SSH store it means that Nix can connect to the specified machine. -When the command succeeds a zero exit code is returned with no output. +If the command succeeds, Nix returns a exit code of 0 and does not +print any output. )"" From 9432c170e736a6b506d9b35ced5eccff6422ec50 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 17:12:58 +0100 Subject: [PATCH 056/725] Fix the drv output map for non ca derivations With the `ca-derivation` experimental features, non-ca derivations used to have their output paths returned as unknown as long as they weren't built (because of a mistake in the code that systematically erased the previous value) --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c61f34275..ab78f1435 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -914,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); else - outputs.insert_or_assign(outputName, std::nullopt); + outputs.insert({outputName, std::nullopt}); } return outputs; From 11b63740e377202e237b7bc74806b82a1eb8ce11 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 21:26:29 +0100 Subject: [PATCH 057/725] Fix content-addressed flake outputs Prevent some `nix flake` commands to crash by trying to parse a placeholder output as a store path --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 50e3b29c4..34ee238bf 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -501,7 +501,7 @@ std::tuple InstallableF auto drvInfo = DerivationInfo{ std::move(drvPath), - state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), attr->getAttr(state->sOutputName)->getString() }; From ea756b3654931f23839aee9f461a8c891c6ffe43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 14:38:31 +0100 Subject: [PATCH 058/725] --refresh: Imply setting .narinfo disk cache TTL to 0 --- src/libstore/nar-info-disk-cache.cc | 6 ++++-- src/nix/main.cc | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..1d8d2d57e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -109,8 +109,10 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) + // Use a minimum TTL to prevent --refresh from + // nuking the entire disk cache. + (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U)) + (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); diff --git a/src/nix/main.cc b/src/nix/main.cc index 418396280..80422bd24 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -330,8 +330,11 @@ void mainWrapped(int argc, char * * argv) fileTransferSettings.connectTimeout = 1; } - if (args.refresh) + if (args.refresh) { settings.tarballTtl = 0; + settings.ttlNegativeNarInfoCache = 0; + settings.ttlPositiveNarInfoCache = 0; + } args.command->second->prepare(); args.command->second->run(); From 555940f0659e95de7f890ede48e2faba096b3d6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 22:50:39 +0100 Subject: [PATCH 059/725] Use enumerate() --- src/nix/build.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 4cb8ade08..45f63bb7e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (size_t i = 0; i < buildables.size(); ++i) + for (const auto & [i, buildable] : enumerate(buildables)) { std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; @@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildables[i]); + }, buildable); + } updateProfile(buildables); From bc90252cec9af05b897cf209012d44a9b20ea251 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 23:08:58 +0100 Subject: [PATCH 060/725] nix profile install: Support installing non-flakes Fixes #4458. --- src/nix/profile.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ca95817d0..765d6866e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -252,8 +252,28 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME manifest.elements.emplace_back(std::move(element)); - } else - throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what()); + } else { + auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + + for (auto & buildable : buildables) { + ProfileElement element; + + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path, {}}); + element.storePaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + pathsToBuild.push_back({bfd.drvPath, {output.first}}); + element.storePaths.insert(output.second); + } + }, + }, buildable); + + manifest.elements.emplace_back(std::move(element)); + } + } } store->buildPaths(pathsToBuild); From 259100332f96250d6615d5839f6a77798c77aefb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 10:29:51 +0100 Subject: [PATCH 061/725] Fix clang build --- src/nix/build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 45f63bb7e..724ce9d79 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -58,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (const auto & [i, buildable] : enumerate(buildables)) { + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; From 144cad906991015e997a6b3e7cc69412eb2b8ab1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 18 Jan 2021 18:13:07 +0100 Subject: [PATCH 062/725] narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash This change is to simplify [Trustix](https://github.com/tweag/trustix) indexing and makes it possible to reconstruct this URL regardless of the compression used. In particular this means that https://github.com/tweag/trustix/blob/7c2e9ca597de233846e0b265fb081626ca6c59d8/contrib/nix/nar/nar.go#L61-L71 can be removed and only the bits that are required to establish trust needs to be published in the Trustix build logs. --- src/libstore/binary-cache-store.cc | 6 +----- tests/binary-cache.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4f5f8607d..15163ead5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,11 +176,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); + narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 355a37d97..937585d6f 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -55,7 +55,7 @@ basicTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) +nar=$(ls $cacheDir/nar/*.nar | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar From 8d4268d1901452164b3e666f2eb6bd6bf516493b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:27:36 +0100 Subject: [PATCH 063/725] Improve error formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { --- src/build-remote/build-remote.cc | 52 ++++---- src/libexpr/attr-set.hh | 2 +- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.cc | 18 +-- src/libexpr/nixexpr.cc | 2 +- src/libexpr/nixexpr.hh | 2 +- src/libexpr/parser.y | 30 ++--- src/libexpr/primops.cc | 90 ++++++------- src/libexpr/primops/context.cc | 6 +- src/libexpr/primops/fetchMercurial.cc | 4 +- src/libexpr/primops/fetchTree.cc | 6 +- src/libexpr/primops/fromTOML.cc | 2 +- src/libstore/build/derivation-goal.cc | 41 +++--- src/libstore/build/substitution-goal.cc | 7 +- src/libstore/build/worker.cc | 5 +- src/libstore/builtins/buildenv.cc | 10 +- src/libstore/filetransfer.cc | 15 +-- src/libstore/local-store.cc | 32 +---- src/libstore/optimise-store.cc | 17 +-- src/libstore/sqlite.cc | 2 +- src/libutil/error.cc | 169 ++++++++---------------- src/libutil/error.hh | 19 +-- src/libutil/logging.cc | 7 +- src/libutil/serialise.cc | 22 ++- src/libutil/tests/logging.cc | 35 ++--- src/nix-build/nix-build.cc | 7 +- src/nix-env/nix-env.cc | 20 +-- src/nix-store/nix-store.cc | 17 +-- src/nix/daemon.cc | 4 +- src/nix/upgrade-nix.cc | 5 +- src/nix/verify.cc | 19 +-- 31 files changed, 249 insertions(+), 422 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8348d8c91..a4cf91858 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -172,13 +172,14 @@ static int main_build_remote(int argc, char * * argv) else { // build the hint template. - string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)"; - hintstring += "\n%s available machines:"; - hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; + string errorText = + "Failed to find a machine for remote build!\n" + "derivation: %s\nrequired (system, features): (%s, %s)"; + errorText += "\n%s available machines:"; + errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)"; - for (unsigned int i = 0; i < machines.size(); ++i) { - hintstring += "\n(%s, %s, %s, %s)"; - } + for (unsigned int i = 0; i < machines.size(); ++i) + errorText += "\n(%s, %s, %s, %s)"; // add the template values. string drvstr; @@ -187,25 +188,21 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto hint = hintformat(hintstring); - hint - % drvstr - % neededSystem - % concatStringsSep(", ", requiredFeatures) - % machines.size(); + auto error = hintformat(errorText); + error + % drvstr + % neededSystem + % concatStringsSep(", ", requiredFeatures) + % machines.size(); - for (auto & m : machines) { - hint % concatStringsSep>(", ", m.systemTypes) - % m.maxJobs - % concatStringsSep(", ", m.supportedFeatures) - % concatStringsSep(", ", m.mandatoryFeatures); - } + for (auto & m : machines) + error + % concatStringsSep>(", ", m.systemTypes) + % m.maxJobs + % concatStringsSep(", ", m.supportedFeatures) + % concatStringsSep(", ", m.mandatoryFeatures); - logErrorInfo(canBuildLocally ? lvlChatty : lvlWarn, { - .name = "Remote build", - .description = "Failed to find a machine for remote build!", - .hint = hint - }); + printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } @@ -230,12 +227,9 @@ static int main_build_remote(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - logError({ - .name = "Remote build", - .hint = hintfmt("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)) - }); + printError("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7eaa16c59..6d68e5df3 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -77,7 +77,7 @@ public: auto a = get(name); if (!a) throw Error({ - .hint = hintfmt("attribute '%s' missing", name), + .msg = hintfmt("attribute '%s' missing", name), .errPos = pos }); diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f6dead6b0..655408cd3 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -10,7 +10,7 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { throw EvalError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } @@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { throw TypeError({ - .hint = hintfmt(s, showType(v)), + .msg = hintfmt(s, showType(v)), .errPos = pos }); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f3471aac7..7271776eb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -622,7 +622,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { throw EvalError({ - .hint = hintfmt(s, s2), + .msg = hintfmt(s, s2), .errPos = pos }); } @@ -635,7 +635,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { throw EvalError({ - .hint = hintfmt(s, s2, s3), + .msg = hintfmt(s, s2, s3), .errPos = pos }); } @@ -644,7 +644,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const { // p1 is where the error occurred; p2 is a position mentioned in the message. throw EvalError({ - .hint = hintfmt(s, sym, p2), + .msg = hintfmt(s, sym, p2), .errPos = p1 }); } @@ -652,7 +652,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { throw TypeError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } @@ -660,7 +660,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ - .hint = hintfmt(s, fun.showNamePos(), s2), + .msg = hintfmt(s, fun.showNamePos(), s2), .errPos = pos }); } @@ -668,7 +668,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { throw AssertionError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -676,7 +676,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { throw UndefinedVarError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -684,7 +684,7 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) { throw MissingArgumentError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -2057,7 +2057,7 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError({ - .hint = hintfmt("cannot coerce %1% to a string", showType()), + .msg = hintfmt("cannot coerce %1% to a string", showType()), .errPos = pos }); } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d5698011f..492b819e7 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env) "undefined variable" error now. */ if (withLevel == -1) throw UndefinedVarError({ - .hint = hintfmt("undefined variable '%1%'", name), + .msg = hintfmt("undefined variable '%1%'", name), .errPos = pos }); fromWith = true; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 530202ff6..cbe9a45bf 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -239,7 +239,7 @@ struct ExprLambda : Expr { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .msg = hintfmt("duplicate formal function argument '%1%'", arg), .errPos = pos }); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 85eb05d61..49d995bb9 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -32,7 +32,7 @@ namespace nix { Path basePath; Symbol file; FileOrigin origin; - ErrorInfo error; + std::optional error; Symbol sLetBody; ParseData(EvalState & state) : state(state) @@ -66,8 +66,8 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(attrPath), prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), .errPos = pos }); } @@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), .errPos = pos }); } @@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", + .msg = hintfmt("duplicate formal function argument '%1%'", formal.name), .errPos = pos }); @@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { data->error = { - .hint = hintfmt(error), + .msg = hintfmt(error), .errPos = makeCurPos(*loc, data) }; } @@ -338,7 +338,7 @@ expr_function | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in let"), + .msg = hintfmt("dynamic attributes not allowed in let"), .errPos = CUR_POS }); $$ = new ExprLet($2, $4); @@ -418,7 +418,7 @@ expr_simple static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) throw ParseError({ - .hint = hintfmt("URL literals are disabled"), + .msg = hintfmt("URL literals are disabled"), .errPos = CUR_POS }); $$ = new ExprString(data->symbols.create($1)); @@ -491,7 +491,7 @@ attrs delete str; } else throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in inherit"), + .msg = hintfmt("dynamic attributes not allowed in inherit"), .errPos = makeCurPos(@2, data) }); } @@ -576,7 +576,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, ParseData data(*this); data.origin = origin; switch (origin) { - case foFile: + case foFile: data.file = data.symbols.create(path); break; case foStdin: @@ -593,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) throw ParseError(data.error); + if (res) throw ParseError(data.error.value()); data.result->bindVars(staticEnv); @@ -703,7 +703,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos return corepkgsPrefix + path.substr(4); throw ThrownError({ - .hint = hintfmt(evalSettings.pureEval + .msg = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), @@ -725,8 +725,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ - .name = "Entry download", - .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } @@ -736,8 +735,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl res = { true, path }; else { logWarning({ - .name = "Entry not found", - .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c73a94f4e..a470ed6df 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -115,7 +115,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -282,7 +282,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -322,7 +322,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) { throw EvalError({ - .hint = hintfmt("at least one argument to 'exec' required"), + .msg = hintfmt("at least one argument to 'exec' required"), .errPos = pos }); } @@ -336,7 +336,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", program, e.path), .errPos = pos }); @@ -551,7 +551,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'startSet' required"), + .msg = hintfmt("attribute 'startSet' required"), .errPos = pos }); state.forceList(*startSet->value, pos); @@ -565,7 +565,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'operator' required"), + .msg = hintfmt("attribute 'operator' required"), .errPos = pos }); state.forceValue(*op->value, pos); @@ -587,7 +587,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'key' required"), + .msg = hintfmt("attribute 'key' required"), .errPos = pos }); state.forceValue(*key->value, pos); @@ -810,7 +810,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) throw EvalError({ - .hint = hintfmt("required attribute 'name' missing"), + .msg = hintfmt("required attribute 'name' missing"), .errPos = pos }); string drvName; @@ -859,7 +859,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else throw EvalError({ - .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .errPos = posDrvName }); }; @@ -869,7 +869,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : ss) { if (outputs.find(j) != outputs.end()) throw EvalError({ - .hint = hintfmt("duplicate derivation output '%1%'", j), + .msg = hintfmt("duplicate derivation output '%1%'", j), .errPos = posDrvName }); /* !!! Check whether j is a valid attribute @@ -879,14 +879,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * the resulting set. */ if (j == "drv") throw EvalError({ - .hint = hintfmt("invalid derivation output name 'drv'" ), + .msg = hintfmt("invalid derivation output name 'drv'" ), .errPos = posDrvName }); outputs.insert(j); } if (outputs.empty()) throw EvalError({ - .hint = hintfmt("derivation cannot have an empty set of outputs"), + .msg = hintfmt("derivation cannot have an empty set of outputs"), .errPos = posDrvName }); }; @@ -1007,20 +1007,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") throw EvalError({ - .hint = hintfmt("required attribute 'builder' missing"), + .msg = hintfmt("required attribute 'builder' missing"), .errPos = posDrvName }); if (drv.platform == "") throw EvalError({ - .hint = hintfmt("required attribute 'system' missing"), + .msg = hintfmt("required attribute 'system' missing"), .errPos = posDrvName }); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) throw EvalError({ - .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .errPos = posDrvName }); @@ -1031,7 +1031,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error({ - .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), .errPos = posDrvName }); @@ -1211,7 +1211,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) throw EvalError({ - .hint = hintfmt("path '%1%' is not in the Nix store", path), + .msg = hintfmt("path '%1%' is not in the Nix store", path), .errPos = pos }); auto path2 = state.store->toStorePath(path).first; @@ -1247,7 +1247,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot check the existence of '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -1324,7 +1324,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1363,7 +1363,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'path' missing"), + .msg = hintfmt("attribute 'path' missing"), .errPos = pos }); @@ -1374,7 +1374,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1400,7 +1400,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va std::optional ht = parseHashType(type); if (!ht) throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash type '%1%'", type), .errPos = pos }); @@ -1430,7 +1430,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val state.realiseContext(ctx); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1650,7 +1650,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') throw EvalError( { - .hint = hintfmt( + .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", name, path), @@ -1801,14 +1801,14 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = pos }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) throw TypeError({ - .hint = hintfmt( + .msg = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", showType(*args[0])), .errPos = pos @@ -1875,7 +1875,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = *attr.pos }); } else if (attr.name == state.sName) @@ -1889,13 +1889,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else throw EvalError({ - .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), .errPos = *attr.pos }); } if (path.empty()) throw EvalError({ - .hint = hintfmt("'path' required"), + .msg = hintfmt("'path' required"), .errPos = pos }); if (name.empty()) @@ -2010,7 +2010,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute '%1%' missing", attr), + .msg = hintfmt("attribute '%1%' missing", attr), .errPos = pos }); // !!! add to stack trace? @@ -2142,7 +2142,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) throw TypeError({ - .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), .errPos = pos }); string name = state.forceStringNoCtx(*j->value, pos); @@ -2152,7 +2152,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) throw TypeError({ - .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), .errPos = pos }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); @@ -2258,7 +2258,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } if (!args[0]->isLambda()) throw TypeError({ - .hint = hintfmt("'functionArgs' requires a function"), + .msg = hintfmt("'functionArgs' requires a function"), .errPos = pos }); @@ -2352,7 +2352,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error({ - .hint = hintfmt("list index %1% is out of bounds", n), + .msg = hintfmt("list index %1% is out of bounds", n), .errPos = pos }); state.forceValue(*list.listElems()[n], pos); @@ -2400,7 +2400,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value state.forceList(*args[0], pos); if (args[0]->listSize() == 0) throw Error({ - .hint = hintfmt("'tail' called on an empty list"), + .msg = hintfmt("'tail' called on an empty list"), .errPos = pos }); @@ -2639,7 +2639,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val if (len < 0) throw EvalError({ - .hint = hintfmt("cannot create list of size %1%", len), + .msg = hintfmt("cannot create list of size %1%", len), .errPos = pos }); @@ -2890,7 +2890,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixFloat f2 = state.forceFloat(*args[1], pos); if (f2 == 0) throw EvalError({ - .hint = hintfmt("division by zero"), + .msg = hintfmt("division by zero"), .errPos = pos }); @@ -2902,7 +2902,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) throw EvalError({ - .hint = hintfmt("overflow in integer division"), + .msg = hintfmt("overflow in integer division"), .errPos = pos }); @@ -3033,7 +3033,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V if (start < 0) throw EvalError({ - .hint = hintfmt("negative start position in 'substring'"), + .msg = hintfmt("negative start position in 'substring'"), .errPos = pos }); @@ -3084,7 +3084,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, std::optional ht = parseHashType(type); if (!ht) throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash type '%1%'", type), .errPos = pos }); @@ -3148,12 +3148,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } @@ -3256,12 +3256,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } @@ -3341,7 +3341,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) throw EvalError({ - .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), .errPos = pos }); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index b570fca31..31cf812b4 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) throw EvalError({ - .hint = hintfmt("Context key '%s' is not a store path", i.name), + .msg = hintfmt("Context key '%s' is not a store path", i.name), .errPos = *i.pos }); if (!settings.readOnlyMode) @@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } @@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 845a1ed1b..4830ebec3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -38,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), .errPos = *attr.pos }); } if (url.empty()) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ab80be2d3..48598acaf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -115,7 +115,7 @@ static void fetchTree( if (!attrs.count("type")) throw Error({ - .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .errPos = pos }); @@ -177,14 +177,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), .errPos = *attr.pos }); } if (!url) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); } else diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 77bff44ae..4c6682dfd 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -82,7 +82,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { throw EvalError({ - .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos }); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..c733ccf08 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -87,8 +87,8 @@ void handleDiffHook( printError(chomp(diffRes.second)); } catch (Error & error) { ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); + // FIXME: wrap errors. + ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } @@ -439,12 +439,9 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); + printError( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); @@ -877,9 +874,12 @@ void DerivationGoal::buildDone() statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; + msg += fmt(";\nlast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } } if (diskFull) @@ -1055,12 +1055,9 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); + printError( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); worker.hook = 0; return rpDecline; } else @@ -3068,10 +3065,7 @@ void DerivationGoal::registerOutputs() auto rewriteOutput = [&]() { /* Apply hash rewriting if necessary. */ if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); + warn("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: this is in-memory. */ StringSink sink; @@ -3359,10 +3353,7 @@ void DerivationGoal::registerOutputs() if (settings.enforceDeterminism) throw NotDeterministic(hint); - logError({ - .name = "Output determinism error", - .hint = hint - }); + printError(hint); curRound = nrRounds; // we know enough, bail out early } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..760fd8ab8 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,11 +146,8 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); + warn("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)); tryNext(); return; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..880a93b15 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -454,10 +454,7 @@ bool Worker::pathContentsGood(const StorePath & path) } pathContentsGoodCache.insert_or_assign(path, res); if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); + printError("path '%s' is corrupted or missing!", store.printStorePath(path)); return res; } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 802fb87bc..e88fc687a 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - directory", - .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) - }); + warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } throw; @@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - skipping symlink", - .hint = hintfmt("skipping dangling symlink '%s'", dstFile) - }); + warn("skipping dangling symlink '%s'", dstFile); continue; } throw; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..677ad44cc 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -632,11 +632,7 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - logError({ - .name = "File transfer", - .hint = hintfmt("unexpected error in download thread: %s", - e.what()) - }); + printError("unexpected error in download thread: %s", e.what()); } { @@ -852,11 +848,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. - if (response && (response->size() < 1024 || response->find("") != string::npos)) { - err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); - } else { - err.hint = hf; - } + if (response && (response->size() < 1024 || response->find("") != string::npos)) + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + else + err.msg = hf; } bool isUri(const string & s) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ab78f1435..f306d8505 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -150,12 +150,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - logError({ - .name = "'build-users-group' not found", - .hint = hintfmt( - "warning: the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup) - }); + printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) @@ -1403,12 +1398,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - logError({ - .name = "Invalid hash", - .hint = hintfmt( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash) - }); + printError("link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash); if (repair) { if (unlink(linkPath.c_str()) == 0) printInfo("removed link '%s'", linkPath); @@ -1441,11 +1432,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - logError({ - .name = "Invalid hash - path modified", - .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -1496,10 +1484,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - logError({ - .name = "Nix path not found", - .hint = hintfmt("path '%s' is not in the Nix store", pathS) - }); + printError("path '%s' is not in the Nix store", pathS); return; } @@ -1522,10 +1507,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - logError({ - .name = "Missing path with referrers", - .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) - }); + printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { repairPath(path); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a0d482ddf..78d587139 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -126,16 +126,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - logWarning({ - .name = "Suspicious file", - .hint = hintfmt("skipping suspicious writable file '%1%'", path) - }); + warn("skipping suspicious writable file '%1%'", path); return; } /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2); return; } @@ -191,10 +188,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - logWarning({ - .name = "Corrupted link", - .hint = hintfmt("removing corrupted link '%1%'", linkPath) - }); + warn("removing corrupted link '%s'", linkPath); unlink(linkPath.c_str()); goto retry; } @@ -229,10 +223,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - logError({ - .name = "Unlink error", - .hint = hintfmt("unable to unlink '%1%'", tempLink) - }); + printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index f5935ee5c..447b4179b 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -211,7 +211,7 @@ void handleSQLiteBusy(const SQLiteBusy & e) lastWarned = now; logWarning({ .name = "Sqlite busy", - .hint = hintfmt(e.what()) + .msg = hintfmt(e.what()) }); } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7dc3f1d3..bc5f9e440 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -204,168 +204,109 @@ void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) } } +static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s) +{ + std::string res; + bool first = true; + + while (!s.empty()) { + auto end = s.find('\n'); + if (!first) res += "\n"; + res += first ? indentFirst : indentRest; + res += s.substr(0, end); + first = false; + if (end == s.npos) break; + s = s.substr(end + 1); + } + + return res; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { - auto errwidth = std::max(getWindowSize().second, 20); - string prefix = ""; - - string levelString; + std::string prefix; switch (einfo.level) { case Verbosity::lvlError: { - levelString = ANSI_RED; - levelString += "error:"; - levelString += ANSI_NORMAL; + prefix = ANSI_RED "error"; + break; + } + case Verbosity::lvlNotice: { + prefix = ANSI_RED "note"; break; } case Verbosity::lvlWarn: { - levelString = ANSI_YELLOW; - levelString += "warning:"; - levelString += ANSI_NORMAL; + prefix = ANSI_YELLOW "warning"; break; } case Verbosity::lvlInfo: { - levelString = ANSI_GREEN; - levelString += "info:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "info"; break; } case Verbosity::lvlTalkative: { - levelString = ANSI_GREEN; - levelString += "talk:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "talk"; break; } case Verbosity::lvlChatty: { - levelString = ANSI_GREEN; - levelString += "chat:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "chat"; break; } case Verbosity::lvlVomit: { - levelString = ANSI_GREEN; - levelString += "vomit:"; - levelString += ANSI_NORMAL; + prefix = ANSI_GREEN "vomit"; break; } case Verbosity::lvlDebug: { - levelString = ANSI_YELLOW; - levelString += "debug:"; - levelString += ANSI_NORMAL; - break; - } - default: { - levelString = fmt("invalid error level: %1%", einfo.level); + prefix = ANSI_YELLOW "debug"; break; } + default: + assert(false); } - auto ndl = prefix.length() - + filterANSIEscapes(levelString, true).length() - + 7 - + einfo.name.length() - + einfo.programName.value_or("").length(); - auto dashwidth = std::max(errwidth - ndl, 3); - - std::string dashes(dashwidth, '-'); - - // divider. - if (einfo.name != "") - out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, - prefix, - levelString, - einfo.name, - dashes, - einfo.programName.value_or("")); + // FIXME: show the program name as part of the trace? + if (einfo.programName && einfo.programName != ErrorInfo::programName) + prefix += fmt(" [%s]:" ANSI_NORMAL " ", einfo.programName.value_or("")); else - out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, - prefix, - levelString, - dashes, - einfo.programName.value_or("")); + prefix += ":" ANSI_NORMAL " "; - bool nl = false; // intersperse newline between sections. - if (einfo.errPos.has_value() && (*einfo.errPos)) { - out << prefix << std::endl; - printAtPos(prefix, *einfo.errPos, out); - nl = true; - } + std::ostringstream oss; + oss << einfo.msg << "\n"; - // description - if (einfo.description != "") { - if (nl) - out << std::endl << prefix; - out << std::endl << prefix << einfo.description; - nl = true; - } + if (einfo.errPos.has_value() && *einfo.errPos) { + oss << "\n"; + printAtPos("", *einfo.errPos, oss); - if (einfo.errPos.has_value() && (*einfo.errPos)) { auto loc = getCodeLines(*einfo.errPos); // lines of code. if (loc.has_value()) { - if (nl) - out << std::endl << prefix; - printCodeLines(out, prefix, *einfo.errPos, *loc); - nl = true; + oss << "\n"; + printCodeLines(oss, "", *einfo.errPos, *loc); + oss << "\n"; } } - // hint - if (einfo.hint.has_value()) { - if (nl) - out << std::endl << prefix; - out << std::endl << prefix << *einfo.hint; - nl = true; - } - // traces - if (showTrace && !einfo.traces.empty()) - { - const string tracetitle(" show-trace "); - - int fill = errwidth - tracetitle.length(); - int lw = 0; - int rw = 0; - const int min_dashes = 3; - if (fill > min_dashes * 2) { - if (fill % 2 != 0) { - lw = fill / 2; - rw = lw + 1; - } - else - { - lw = rw = fill / 2; - } - } - else - lw = rw = min_dashes; - - if (nl) - out << std::endl << prefix; - - out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL; - - for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) - { - out << std::endl << prefix; - out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str(); + if (showTrace && !einfo.traces.empty()) { + for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { + oss << "\n" << "… " << iter->hint.str() << "\n"; if (iter->pos.has_value() && (*iter->pos)) { auto pos = iter->pos.value(); - out << std::endl << prefix; - printAtPos(prefix, pos, out); + oss << "\n"; + printAtPos("", pos, oss); auto loc = getCodeLines(pos); - if (loc.has_value()) - { - out << std::endl << prefix; - printCodeLines(out, prefix, pos, *loc); - out << std::endl << prefix; + if (loc.has_value()) { + oss << "\n"; + printCodeLines(oss, "", pos, *loc); + oss << "\n"; } } } } + out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str())); + return out; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e0bde7ea..ff58d3e00 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -107,9 +107,8 @@ struct Trace { struct ErrorInfo { Verbosity level; - string name; - string description; // FIXME: remove? it seems to be barely used - std::optional hint; + string name; // FIXME: rename + hintformat msg; std::optional errPos; std::list traces; @@ -133,23 +132,17 @@ public: template BaseError(unsigned int status, const Args & ... args) - : err {.level = lvlError, - .hint = hintfmt(args...) - } + : err { .level = lvlError, .msg = hintfmt(args...) } , status(status) { } template BaseError(const std::string & fs, const Args & ... args) - : err {.level = lvlError, - .hint = hintfmt(fs, args...) - } + : err { .level = lvlError, .msg = hintfmt(fs, args...) } { } BaseError(hintformat hint) - : err {.level = lvlError, - .hint = hint - } + : err { .level = lvlError, .msg = hint } { } BaseError(ErrorInfo && e) @@ -206,7 +199,7 @@ public: { errNo = errno; auto hf = hintfmt(args...); - err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } virtual const char* sname() const override { return "SysError"; } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6fd0dacef..d2e801175 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -184,7 +184,7 @@ struct JSONLogger : Logger { json["action"] = "msg"; json["level"] = ei.level; json["msg"] = oss.str(); - json["raw_msg"] = ei.hint->str(); + json["raw_msg"] = ei.msg.str(); if (ei.errPos.has_value() && (*ei.errPos)) { json["line"] = ei.errPos->line; @@ -305,10 +305,7 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - logError({ - .name = "JSON log message", - .hint = hintfmt("bad log message from builder: %s", e.what()) - }); + printError("bad JSON log message from builder: %s", e.what()); } return true; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 87c1099a1..d1a16b6ba 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -52,10 +52,7 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - logWarning({ - .name = "Large path", - .description = "dumping very large path (> 256 MiB); this may run out of memory" - }); + warn("dumping very large path (> 256 MiB); this may run out of memory"); } @@ -306,8 +303,7 @@ Sink & operator << (Sink & sink, const Error & ex) << "Error" << info.level << info.name - << info.description - << (info.hint ? info.hint->str() : "") + << info.msg.str() << 0 // FIXME: info.errPos << info.traces.size(); for (auto & trace : info.traces) { @@ -374,12 +370,14 @@ Error readError(Source & source) { auto type = readString(source); assert(type == "Error"); - ErrorInfo info; - info.level = (Verbosity) readInt(source); - info.name = readString(source); - info.description = readString(source); - auto hint = readString(source); - if (hint != "") info.hint = hintformat(std::move(format("%s") % hint)); + auto level = (Verbosity) readInt(source); + auto name = readString(source); + auto msg = readString(source); + ErrorInfo info { + .level = level, + .name = name, + .msg = hintformat(std::move(format("%s") % msg)), + }; auto havePos = readNum(source); assert(havePos == 0); auto nrTraces = readNum(source); diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 5b32c84a4..d990e5499 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -1,3 +1,5 @@ +#if 0 + #include "logging.hh" #include "nixexpr.hh" #include "util.hh" @@ -41,8 +43,7 @@ namespace nix { makeJSONLogger(*logger)->logEI({ .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -62,7 +63,7 @@ namespace nix { throw TestError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : "")); + ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str())); testing::internal::CaptureStderr(); logger->logEI(ei); @@ -95,7 +96,6 @@ namespace nix { logger->logEI({ .level = lvlInfo, .name = "Info name", - .description = "Info description", }); auto str = testing::internal::GetCapturedStderr(); @@ -109,7 +109,6 @@ namespace nix { logger->logEI({ .level = lvlTalkative, .name = "Talkative name", - .description = "Talkative description", }); auto str = testing::internal::GetCapturedStderr(); @@ -123,7 +122,6 @@ namespace nix { logger->logEI({ .level = lvlChatty, .name = "Chatty name", - .description = "Talkative description", }); auto str = testing::internal::GetCapturedStderr(); @@ -137,7 +135,6 @@ namespace nix { logger->logEI({ .level = lvlDebug, .name = "Debug name", - .description = "Debug description", }); auto str = testing::internal::GetCapturedStderr(); @@ -151,7 +148,6 @@ namespace nix { logger->logEI({ .level = lvlVomit, .name = "Vomit name", - .description = "Vomit description", }); auto str = testing::internal::GetCapturedStderr(); @@ -167,7 +163,6 @@ namespace nix { logError({ .name = "name", - .description = "error description", }); auto str = testing::internal::GetCapturedStderr(); @@ -182,8 +177,7 @@ namespace nix { logError({ .name = "error name", - .description = "error with code lines", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foString, problem_file, 02, 13), @@ -200,8 +194,7 @@ namespace nix { logError({ .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -216,7 +209,7 @@ namespace nix { logError({ .name = "error name", - .hint = hintfmt("hint %1%", "only"), + .msg = hintfmt("hint %1%", "only"), }); auto str = testing::internal::GetCapturedStderr(); @@ -233,8 +226,7 @@ namespace nix { logWarning({ .name = "name", - .description = "warning description", - .hint = hintfmt("there was a %1%", "warning"), + .msg = hintfmt("there was a %1%", "warning"), }); auto str = testing::internal::GetCapturedStderr(); @@ -250,8 +242,7 @@ namespace nix { logWarning({ .name = "warning name", - .description = "warning description", - .hint = hintfmt("this hint has %1% templated %2%!!", + .msg = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foStdin, problem_file, 2, 13), @@ -274,8 +265,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "show-traces", - .hint = hintfmt("it has been %1% days since our last error", "zero"), + .msg = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -301,8 +291,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .description = "hide traces", - .hint = hintfmt("it has been %1% days since our last error", "zero"), + .msg = hintfmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -377,3 +366,5 @@ namespace nix { } } + +#endif diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 38048da52..d1c14596c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -369,11 +369,8 @@ static void main_nix_build(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - logWarning({ - .name = "bashInteractive", - .hint = hintfmt("%s; will use bash from your environment", - (e.info().hint ? e.info().hint->str() : "")) - }); + logError(e.info()); + notice("will use bash from your environment"); shell = "bash"; } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 9963f05d9..d6a16999f 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -124,10 +124,7 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - logError({ - .name = "Name collision", - .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) - }); + printError("warning: name collision in input Nix expressions, skipping '%1%'", path2); continue; } /* Load the expression on demand. */ @@ -876,11 +873,7 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - logError({ - .name = "Invalid meta attribute", - .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) - }); + printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); placeholder.write(nullptr); } else { PathSet context; @@ -1131,12 +1124,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["name"] = j; Value * v = i.queryMeta(j); if (!v) - logError({ - .name = "Invalid meta attribute", - .hint = hintfmt( - "derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) - }); + printError( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); else { if (v->type() == nString) { attrs2["type"] = "string"; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b97f684a4..b7eda5ba6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -708,10 +708,7 @@ static void opVerify(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - logWarning({ - .name = "Store consistency", - .description = "not all errors were fixed" - }); + warn("not all store errors were fixed"); throw Exit(1); } } @@ -733,14 +730,10 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - logError({ - .name = "Hash mismatch", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(Base32, true), + current.first.to_string(Base32, true)); status = 1; } } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 204d4ce6b..a358cb0d9 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -258,8 +258,8 @@ static void daemonLoop() return; } catch (Error & error) { ErrorInfo ei = error.info(); - ei.hint = std::optional(hintfmt("error processing connection: %1%", - (error.info().hint.has_value() ? error.info().hint->str() : ""))); + // FIXME: add to trace? + ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); logError(ei); } } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 299ea40aa..9cd567896 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -61,10 +61,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - logWarning({ - .name = "Version update", - .hint = hintfmt("would upgrade to version %s", version) - }); + warn("would upgrade to version %s", version); return; } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index b2963cf74..9b04e032a 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -101,14 +101,10 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - logError({ - .name = "Hash error - path modified", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(Base32, true), + hash.first.to_string(Base32, true)); } } @@ -156,12 +152,7 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - logError({ - .name = "Untrusted path", - .hint = hintfmt("path '%s' is untrusted", - store->printStorePath(info->path)) - }); - + printError("path '%s' is untrusted", store->printStorePath(info->path)); } } From 40608342cb3772a6d2a6c125cc2237b97c028ab4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:49:29 +0100 Subject: [PATCH 064/725] Remove trailing whitespace --- src/libutil/error.cc | 3 +-- src/libutil/util.cc | 2 +- src/libutil/util.hh | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index bc5f9e440..ddeb5412a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -212,8 +212,7 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR while (!s.empty()) { auto end = s.find('\n'); if (!first) res += "\n"; - res += first ? indentFirst : indentRest; - res += s.substr(0, end); + res += chomp(std::string(first ? indentFirst : indentRest) + std::string(s.substr(0, end))); first = false; if (end == s.npos) break; s = s.substr(end + 1); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e6b6d287d..89f7b58f8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1249,7 +1249,7 @@ template StringSet tokenizeString(std::string_view s, const string & separators) template vector tokenizeString(std::string_view s, const string & separators); -string chomp(const string & s) +string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); return i == string::npos ? "" : string(s, 0, i + 1); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ab0bd865a..ad49c65b3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -373,8 +373,9 @@ template Strings quoteStrings(const C & c) } -/* Remove trailing whitespace from a string. */ -string chomp(const string & s); +/* Remove trailing whitespace from a string. FIXME: return + std::string_view. */ +string chomp(std::string_view s); /* Remove whitespace from the start and end of a string. */ From 55849e153e4b28d03bfca1738c415c438c60f9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:55:59 +0100 Subject: [PATCH 065/725] Change error position formatting It's now at /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix:7:7: instead of at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix The new format is more standard and clickable. --- src/libutil/error.cc | 22 +++++++++------------- tests/misc.sh | 4 ++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index ddeb5412a..5d570a75e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -43,9 +43,9 @@ string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { - return fmt("(%1%:%2%)", errPos.line, errPos.column); + return fmt("%d:%d", errPos.line, errPos.column); } else { - return fmt("(%1%)", errPos.line); + return fmt("%d", errPos.line); } } else { @@ -178,24 +178,20 @@ void printCodeLines(std::ostream & out, } } -void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out) +void printAtPos(const ErrPos & pos, std::ostream & out) { - if (pos) - { + if (pos) { switch (pos.origin) { case foFile: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); break; } case foString: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " from string" << ANSI_NORMAL; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } case foStdin: { - out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) << - ANSI_BLUE << " from stdin" << ANSI_NORMAL; + out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } default: @@ -272,7 +268,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (einfo.errPos.has_value() && *einfo.errPos) { oss << "\n"; - printAtPos("", *einfo.errPos, oss); + printAtPos(*einfo.errPos, oss); auto loc = getCodeLines(*einfo.errPos); @@ -292,7 +288,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (iter->pos.has_value() && (*iter->pos)) { auto pos = iter->pos.value(); oss << "\n"; - printAtPos("", pos, oss); + printAtPos(pos, oss); auto loc = getCodeLines(pos); if (loc.has_value()) { diff --git a/tests/misc.sh b/tests/misc.sh index a81c9dbb1..2830856ae 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -17,10 +17,10 @@ nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) -echo $eval_arg_res | grep "at: (1:15) from string" +echo $eval_arg_res | grep "at «string»:1:15:" echo $eval_arg_res | grep "infinite recursion encountered" eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true) -echo $eval_stdin_res | grep "at: (1:15) from stdin" +echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" From 0eb22db3116585821096b7b81295d4bbf5550343 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 12:46:22 +0100 Subject: [PATCH 066/725] Fix macOS build --- .../resolve-system-dependencies.cc | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index d30227e4e..27cf53a45 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,18 +39,12 @@ std::set runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - logError({ - .name = "Regular MACH file", - .hint = hintfmt("file '%s' is not a regular file", filename) - }); + printError("file '%s' is not a regular MACH binary", filename); return {}; } if (st.st_size < sizeof(mach_header_64)) { - logError({ - .name = "File too short", - .hint = hintfmt("file '%s' is too short for a MACH binary", filename) - }); + printError("file '%s' is too short for a MACH binary", filename); return {}; } @@ -72,19 +66,13 @@ std::set runResolver(const Path & filename) } } if (mach64_offset == 0) { - logError({ - .name = "No mach64 blobs", - .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) - }); + printError("could not find any mach64 blobs in file '%1%', continuing...", filename); return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - logError({ - .name = "Magic number", - .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) - }); + printError("Object file has unknown magic number '%1%', skipping it...", magic); return {}; } From d9367a2dd1f2cfe163b9c42e83a0569808ce6fc9 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 21 Jan 2021 17:30:26 +0100 Subject: [PATCH 067/725] scripts/install-nix-from-closure: only show progress if a terminal is used While the progress dots during the copying of the store work fine on a normal terminal, those look pretty off if the script is run inside a provisioning script of e.g. `vagrant` or `packer` where `stderr` and `stdout` are captured: default: . default: .. default: . default: . default: . To work around this, the script checks with `-t 0` if it's running on an actual terminal and doesn't show the progress if that's not the case. --- scripts/install-nix-from-closure.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 6352a8fac..0ee7ce5af 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -166,9 +166,15 @@ fi mkdir -p $dest/store printf "copying Nix to %s..." "${dest}/store" >&2 +# Insert a newline if no progress is shown. +if [ ! -t 0 ]; then + echo "" +fi for i in $(cd "$self/store" >/dev/null && echo ./*); do - printf "." >&2 + if [ -t 0 ]; then + printf "." >&2 + fi i_tmp="$dest/store/$i.$$" if [ -e "$i_tmp" ]; then rm -rf "$i_tmp" From b7bfc7ee52dd425e0156f369eb4c05a62358f912 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 14:54:28 +0100 Subject: [PATCH 068/725] Add FIXME --- src/libutil/error.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7dc3f1d3..2a67a730a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -61,6 +61,8 @@ std::optional getCodeLines(const ErrPos & errPos) if (errPos.origin == foFile) { LinesOfCode loc; try { + // FIXME: when running as the daemon, make sure we don't + // open a file to which the client doesn't have access. AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) return {}; From 8c07ed1ddad6595cd679181b0b8d78e09fc6d152 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 15:27:55 +0000 Subject: [PATCH 069/725] Improve documentation and test and requested --- src/libstore/store-api.hh | 6 +++--- tests/binary-cache-build-remote.sh | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3221cf249..9e98eb8f9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -372,9 +372,9 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently well-formed - (e.g. hash content-address or signature) in order to be included in the - given store. + /* Check whether the given valid path info is sufficiently attested, by + either being signed by a trusted public key or content-addressed, in + order to be included in the given store. These same checks would be performed in addToStore, but this allows an earlier failure in the case where dependencies need to be added too, but diff --git a/tests/binary-cache-build-remote.sh b/tests/binary-cache-build-remote.sh index ed51164a4..81cd21a4a 100644 --- a/tests/binary-cache-build-remote.sh +++ b/tests/binary-cache-build-remote.sh @@ -7,7 +7,10 @@ clearCacheCache (! nix-build --store "file://$cacheDir" dependencies.nix) # Succeeds with default store as build remote. -nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix +outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) + +# Test that the path exactly exists in the destination store. +nix path-info --store "file://$cacheDir" $outPath # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix From 53a709535b42197a9abd3fe46406bb186ad6c751 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 10:21:12 -0500 Subject: [PATCH 070/725] Apply suggestions from code review Thanks! Co-authored-by: Eelco Dolstra --- src/build-remote/build-remote.cc | 6 +++--- src/libstore/build/derivation-goal.cc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 350bd6cef..68af3e966 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -75,11 +75,11 @@ static int main_build_remote(int argc, char * * argv) /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = "/current-load"; + auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoad; + currentLoad = std::string { localStore->stateDir } + currentLoadName; else - currentLoad = settings.nixStateDir + currentLoad; + currentLoad = settings.nixStateDir + currentLoadName; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 953e241d8..fa8b99118 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3291,7 +3291,7 @@ void DerivationGoal::registerOutputs() auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; if (buildMode == bmCheck) { @@ -3426,7 +3426,7 @@ void DerivationGoal::registerOutputs() { auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; ValidPathInfos infos2; From a76682466062ef2c972d19f259feeef1c46a44a3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 14:46:40 -0600 Subject: [PATCH 071/725] Handle missing etag in 304 Not Modified response GitHub now omits the etag, but 304 implies it matches the one we provided. Just use that one to avoid having an etag-less resource. Fixes #4469 --- src/libstore/filetransfer.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..1b7eae3ec 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -375,6 +375,13 @@ struct curlFileTransfer : public FileTransfer else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; + + // In 2021, GitHub responds to If-None-Match with 304, + // but omits ETag. We just use the If-None-Match etag + // since 304 implies they are the same. + if (httpStatus == 304 && result.etag == "") + result.etag = request.expectedETag; + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); From 1ea5f0b66ca43eb1f6c552b59de170d61bcf540c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 23:19:52 -0600 Subject: [PATCH 072/725] Remove expectedETag assert in tarball.cc --- src/libfetchers/tarball.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 56c014a8c..b8d7d2c70 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -64,7 +64,6 @@ DownloadFileResult downloadFile( if (res.cached) { assert(cached); - assert(request.expectedETag == res.etag); storePath = std::move(cached->storePath); } else { StringSink sink; From b159d23800eec55412621a0b3e6c926a1dbb1755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 14:38:15 +0100 Subject: [PATCH 073/725] Make '--help' do the same as 'help' (i.e. show a manpage) --- src/libutil/args.cc | 89 --------------------------------------------- src/libutil/args.hh | 14 ------- src/nix/command.cc | 5 --- src/nix/command.hh | 2 - src/nix/main.cc | 61 +++++++++---------------------- src/nix/nar.cc | 5 --- src/nix/store.cc | 5 --- 7 files changed, 17 insertions(+), 164 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index fb5cb80fb..2f2e4bb96 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -96,41 +96,6 @@ void Args::parseCmdline(const Strings & _cmdline) processArgs(pendingArgs, true); } -void Args::printHelp(const string & programName, std::ostream & out) -{ - std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); - for (auto & exp : expectedArgs) { - std::cout << renderLabels({exp.label}); - // FIXME: handle arity > 1 - if (exp.handler.arity == ArityAny) std::cout << "..."; - if (exp.optional) std::cout << "?"; - } - std::cout << "\n"; - - auto s = description(); - if (s != "") - std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; - - if (longFlags.size()) { - std::cout << "\n"; - std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; - printFlags(out); - } -} - -void Args::printFlags(std::ostream & out) -{ - Table2 table; - for (auto & flag : longFlags) { - if (hiddenCategories.count(flag.second->category)) continue; - table.push_back(std::make_pair( - (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") - + "--" + flag.first + renderLabels(flag.second->labels), - flag.second->description)); - } - printTable(out, table); -} - bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); @@ -331,28 +296,6 @@ Strings argvToStrings(int argc, char * * argv) return args; } -std::string renderLabels(const Strings & labels) -{ - std::string res; - for (auto label : labels) { - for (auto & c : label) c = std::toupper(c); - res += " " ANSI_ITALIC + label + ANSI_NORMAL; - } - return res; -} - -void printTable(std::ostream & out, const Table2 & table) -{ - size_t max = 0; - for (auto & row : table) - max = std::max(max, filterANSIEscapes(row.first, true).size()); - for (auto & row : table) { - out << " " << row.first - << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') - << row.second << "\n"; - } -} - MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { @@ -376,38 +319,6 @@ MultiCommand::MultiCommand(const Commands & commands) categories[Command::catDefault] = "Available commands"; } -void MultiCommand::printHelp(const string & programName, std::ostream & out) -{ - if (command) { - command->second->printHelp(programName + " " + command->first, out); - return; - } - - out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); - - out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; - printFlags(out); - - std::map>> commandsByCategory; - - for (auto & [name, commandFun] : commands) { - auto command = commandFun(); - commandsByCategory[command->category()].insert_or_assign(name, command); - } - - for (auto & [category, commands] : commandsByCategory) { - out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); - - Table2 table; - for (auto & [name, command] : commands) { - auto descr = command->description(); - if (!descr.empty()) - table.push_back(std::make_pair(name, descr)); - } - printTable(out, table); - } -} - bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { if (Args::processFlag(pos, end)) return true; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 3783bc84f..fda7852cd 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -20,8 +20,6 @@ public: wrong. */ void parseCmdline(const Strings & cmdline); - virtual void printHelp(const string & programName, std::ostream & out); - /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } @@ -115,8 +113,6 @@ protected: virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - virtual void printFlags(std::ostream & out); - /* Positional arguments. */ struct ExpectedArg { @@ -223,8 +219,6 @@ public: MultiCommand(const Commands & commands); - void printHelp(const string & programName, std::ostream & out) override; - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; @@ -234,14 +228,6 @@ public: Strings argvToStrings(int argc, char * * argv); -/* Helper function for rendering argument labels. */ -std::string renderLabels(const Strings & labels); - -/* Helper function for printing 2-column tables. */ -typedef std::vector> Table2; - -void printTable(std::ostream & out, const Table2 & table); - struct Completion { std::string completion; std::string description; diff --git a/src/nix/command.cc b/src/nix/command.cc index ba58c7d6b..20eeefe91 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -27,11 +27,6 @@ nix::Commands RegisterCommand::getCommandsFor(const std::vector & p return res; } -void NixMultiCommand::printHelp(const string & programName, std::ostream & out) -{ - MultiCommand::printHelp(programName, out); -} - nlohmann::json NixMultiCommand::toJSON() { // FIXME: use Command::toJSON() as well. diff --git a/src/nix/command.hh b/src/nix/command.hh index f325cd906..791dd0f1e 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -25,8 +25,6 @@ static constexpr Command::Category catNixInstallation = 102; struct NixMultiCommand : virtual MultiCommand, virtual Command { - void printHelp(const string & programName, std::ostream & out) override; - nlohmann::json toJSON() override; }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 80422bd24..77a13c913 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -54,6 +54,8 @@ static bool haveInternet() std::string programPath; char * * savedArgv; +struct HelpRequested { }; + struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; @@ -71,22 +73,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", .description = "Show usage information.", - .handler = {[&]() { if (!completions) showHelpAndExit(); }}, - }); - - addFlag({ - .longName = "help-config", - .description = "Show configuration settings.", - .handler = {[&]() { - std::cout << "The following configuration settings are available:\n\n"; - Table2 tbl; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - tbl.emplace_back(s.first, s.second.description); - printTable(std::cout, tbl); - throw Exit(); - }}, + .handler = {[&]() { throw HelpRequested(); }}, }); addFlag({ @@ -154,33 +141,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs return pos; } - void printFlags(std::ostream & out) override - { - Args::printFlags(out); - std::cout << - "\n" - "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" - "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" - "--help-config' for a list of configuration settings.\n"; - } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - -#if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; -#endif - - std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; - } - - void showHelpAndExit() - { - printHelp(programName, std::cout); - throw Exit(); - } - std::string description() override { return "a tool for reproducible and declarative configuration management"; @@ -298,6 +258,18 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); + } catch (HelpRequested &) { + std::vector subcommand; + MultiCommand * command = &args; + while (command) { + if (command && command->command) { + subcommand.push_back(command->command->first); + command = dynamic_cast(&*command->command->second); + } else + break; + } + showHelp(subcommand); + return; } catch (UsageError &) { if (!completions) throw; } @@ -306,7 +278,8 @@ void mainWrapped(int argc, char * * argv) initPlugins(); - if (!args.command) args.showHelpAndExit(); + if (!args.command) + throw UsageError("no subcommand specified"); if (args.command->first != "repl" && args.command->first != "doctor" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index 0775d3c25..dbb043d9b 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -28,11 +28,6 @@ struct CmdNar : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/store.cc b/src/nix/store.cc index e91bcc503..44e53c7c7 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -21,11 +21,6 @@ struct CmdStore : virtual NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdStore = registerCommand("store"); From a32073e7e839ea92ada602c0a170855a08afc73a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 14:43:16 +0100 Subject: [PATCH 074/725] Add FIXME --- src/libexpr/primops/fetchTree.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 48598acaf..27d8ddf35 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -153,6 +153,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V fetchTree(state, pos, args, v, std::nullopt); } +// FIXME: document static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, From 3ba98ba8f08523e60310cf75ec809bd21d0ce977 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 17:15:38 +0100 Subject: [PATCH 075/725] Tell user to run 'nix log' to get full build logs --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 36bbe46d4..179a010d4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -896,6 +896,8 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } + msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + worker.store.printStorePath(drvPath)); } if (diskFull) From 807d963ee8d23e88f09e28365b045d322530c5aa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 18:19:32 +0100 Subject: [PATCH 076/725] Group subcommands by category --- doc/manual/generate-manpage.nix | 23 +++++++++++++++++++---- doc/manual/utils.nix | 10 +++++++++- src/libutil/args.cc | 5 ++++- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index c2c748464..30152088d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -13,12 +13,27 @@ let + showSynopsis { inherit command; args = def.args; } + (if def.commands or {} != {} then + let + categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands))); + listCommands = cmds: + concatStrings (map (name: + "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n") + (attrNames cmds)); + in "where *subcommand* is one of the following:\n\n" # FIXME: group by category - + concatStrings (map (name: - "* [`${command} ${name}`](./${appendName filename name}.md) - ${def.commands.${name}.description}\n") - (attrNames def.commands)) - + "\n" + + (if length categories > 1 + then + concatStrings (map + (cat: + "**${toString cat.description}:**\n\n" + + listCommands (filterAttrs (n: v: v.category == cat) def.commands) + + "\n" + ) categories) + + "\n" + else + listCommands def.commands + + "\n") else "") + (if def ? doc then def.doc + "\n\n" diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 50150bf3e..d4b18472f 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,7 +1,15 @@ with builtins; -{ +rec { splitLines = s: filter (x: !isList x) (split "\n" s); concatStrings = concatStringsSep ""; + + # FIXME: O(n^2) + unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; + + nameValuePair = name: value: { inherit name value; }; + + filterAttrs = pred: set: + listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 2f2e4bb96..6d57e1a34 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -341,7 +341,10 @@ nlohmann::json MultiCommand::toJSON() for (auto & [name, commandFun] : commands) { auto command = commandFun(); auto j = command->toJSON(); - j["category"] = categories[command->category()]; + auto cat = nlohmann::json::object(); + cat["id"] = command->category(); + cat["description"] = categories[command->category()]; + j["category"] = std::move(cat); cmds[name] = std::move(j); } From 36c4d6f59247826dde32ad2e6b5a9471a9a1c911 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 19:03:13 +0100 Subject: [PATCH 077/725] Group common options --- doc/manual/generate-manpage.nix | 40 ++++++++++++++++++++------------- src/libexpr/common-eval-args.cc | 7 ++++++ src/libmain/common-args.cc | 6 ++++- src/libmain/common-args.hh | 17 ++++++++++++-- src/libutil/args.cc | 3 +-- src/libutil/args.hh | 2 +- src/nix/command.cc | 9 +++++++- src/nix/command.hh | 2 ++ src/nix/installables.cc | 13 +++++++++++ src/nix/main.cc | 1 + src/nix/sigs.cc | 6 ++--- 11 files changed, 80 insertions(+), 26 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 30152088d..a563c31f8 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,31 +38,39 @@ let + (if def ? doc then def.doc + "\n\n" else "") - + (let s = showFlags def.flags; in + + (let s = showOptions def.flags; in if s != "" - then "# Flags\n\n${s}" + then "# Options\n\n${s}" else "") ; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; - showFlags = flags: - concatStrings - (map (longName: - let flag = flags.${longName}; in - if flag.category or "" != "config" - then - " - `--${longName}`" - + (if flag ? shortName then " / `-${flag.shortName}`" else "") - + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") - + " \n" - + " " + flag.description + "\n\n" - else "") - (attrNames flags)); + showOptions = flags: + let + categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags))); + in + concatStrings (map + (cat: + (if cat != "" + then "**${cat}:**\n\n" + else "") + + concatStrings + (map (longName: + let + flag = flags.${longName}; + in + " - `--${longName}`" + + (if flag ? shortName then " / `-${flag.shortName}`" else "") + + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") + + " \n" + + " " + flag.description + "\n\n" + ) (attrNames (filterAttrs (n: v: v.category == cat) flags)))) + categories); showSynopsis = { command, args }: - "`${command}` [*flags*...] ${concatStringsSep " " + "`${command}` [*option*...] ${concatStringsSep " " (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; processCommand = { command, def, filename }: diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index ffe782454..aa14bf79b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -12,9 +12,12 @@ namespace nix { MixEvalArgs::MixEvalArgs() { + auto category = "Common evaluation options"; + addFlag({ .longName = "arg", .description = "Pass the value *expr* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); @@ -22,6 +25,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "argstr", .description = "Pass the string *string* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -30,6 +34,7 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); @@ -37,6 +42,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "impure", .description = "Allow access to mutable paths and repositories.", + .category = category, .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -45,6 +51,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", + .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index bd5573e5d..ff96ee7d5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -11,18 +11,21 @@ MixCommonArgs::MixCommonArgs(const string & programName) .longName = "verbose", .shortName = 'v', .description = "Increase the logging verbosity level.", + .category = loggingCategory, .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", .description = "Decrease the logging verbosity level.", + .category = loggingCategory, .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", .description = "Set the logging verbosity level to 'debug'.", + .category = loggingCategory, .handler = {[]() { verbosity = lvlDebug; }}, }); @@ -52,6 +55,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", + .category = loggingCategory, .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -66,7 +70,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) }} }); - std::string cat = "config"; + std::string cat = "Options to override configuration settings"; globalConfig.convertToArgs(*this, cat); // Backward compatibility hack: nix-env already had a --system flag. diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 47f341619..8e53a7361 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -4,6 +4,9 @@ namespace nix { +//static constexpr auto commonArgsCategory = "Miscellaneous common options"; +static constexpr auto loggingCategory = "Logging-related options"; + struct MixCommonArgs : virtual Args { string programName; @@ -16,7 +19,12 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); + addFlag({ + .longName = "dry-run", + .description = "Show what this command would do without doing it.", + //.category = commonArgsCategory, + .handler = {&dryRun, true}, + }); } }; @@ -26,7 +34,12 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); + addFlag({ + .longName = "json", + .description = "Produce output in JSON format, suitable for consumption by another program.", + //.category = commonArgsCategory, + .handler = {&json, true}, + }); } }; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6d57e1a34..71bae0504 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -195,8 +195,7 @@ nlohmann::json Args::toJSON() j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") j["description"] = flag->description; - if (flag->category != "") - j["category"] = flag->category; + j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; if (!flag->labels.empty()) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index fda7852cd..b1020b101 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -91,7 +91,7 @@ protected: { } }; - /* Flags. */ + /* Options. */ struct Flag { typedef std::shared_ptr ptr; diff --git a/src/nix/command.cc b/src/nix/command.cc index 20eeefe91..614dee788 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -61,6 +61,7 @@ StorePathsCommand::StorePathsCommand(bool recursive) addFlag({ .longName = "no-recursive", .description = "Apply operation to specified paths only.", + .category = installablesCategory, .handler = {&this->recursive, false}, }); else @@ -68,10 +69,16 @@ StorePathsCommand::StorePathsCommand(bool recursive) .longName = "recursive", .shortName = 'r', .description = "Apply operation to closure of the specified paths.", + .category = installablesCategory, .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "Apply the operation to every store path.", &all); + addFlag({ + .longName = "all", + .description = "Apply the operation to every store path.", + .category = installablesCategory, + .handler = {&all, true}, + }); } void StorePathsCommand::run(ref store) diff --git a/src/nix/command.hh b/src/nix/command.hh index 791dd0f1e..ed6980075 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -23,6 +23,8 @@ static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; +static constexpr auto installablesCategory = "Options that change the interpretation of installables"; + struct NixMultiCommand : virtual MultiCommand, virtual Command { nlohmann::json toJSON() override; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 34ee238bf..4e6bf4a9a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -58,39 +58,47 @@ void completeFlakeInputPath( MixFlakeOptions::MixFlakeOptions() { + auto category = "Common flake-related options"; + addFlag({ .longName = "recreate-lock-file", .description = "Recreate the flake's lock file from scratch.", + .category = category, .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", + .category = category, .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", .description = "Do not write the flake's newly generated lock file.", + .category = category, .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", .description = "Don't allow lookups in the flake registries.", + .category = category, .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", .description = "Commit changes to the flake's lock file.", + .category = category, .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -104,6 +112,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -115,6 +124,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", + .category = category, .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -144,6 +154,7 @@ SourceExprCommand::SourceExprCommand() .longName = "file", .shortName = 'f', .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", + .category = installablesCategory, .labels = {"file"}, .handler = {&file}, .completer = completePath @@ -152,6 +163,7 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "expr", .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", + .category = installablesCategory, .labels = {"expr"}, .handler = {&expr} }); @@ -159,6 +171,7 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "derivation", .description = "Operate on the store derivation rather than its outputs.", + .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 77a13c913..58b643cc5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -80,6 +80,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "print-build-logs", .shortName = 'L', .description = "Print full build logs on standard error.", + .category = loggingCategory, .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 3445182f2..c64b472b6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "Use signatures from specified store.", + .description = "Copy signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -24,7 +24,7 @@ struct CmdCopySigs : StorePathsCommand std::string description() override { - return "copy path signatures from substituters (like binary caches)"; + return "copy store path signatures from substituters"; } void run(ref store, StorePaths storePaths) override @@ -110,7 +110,7 @@ struct CmdSign : StorePathsCommand std::string description() override { - return "sign the specified paths"; + return "sign store paths"; } void run(ref store, StorePaths storePaths) override From f15f0b8e83051cd95dacb2784b004c8272957f30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Jan 2021 10:34:59 +0100 Subject: [PATCH 078/725] Update to lowdown 0.7.9 --- flake.lock | 17 ----------------- flake.nix | 14 ++++++-------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/flake.lock b/flake.lock index 9f8c788ac..6fe52fbfd 100644 --- a/flake.lock +++ b/flake.lock @@ -1,21 +1,5 @@ { "nodes": { - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1598695561, - "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1602702596, @@ -33,7 +17,6 @@ }, "root": { "inputs": { - "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 9addccd63..fedd0e381 100644 --- a/flake.nix +++ b/flake.nix @@ -2,9 +2,9 @@ description = "The purely functional package manager"; inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; + #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; - outputs = { self, nixpkgs, lowdown-src }: + outputs = { self, nixpkgs }: let @@ -200,16 +200,14 @@ }; lowdown = with final; stdenv.mkDerivation { - name = "lowdown-0.7.1"; + name = "lowdown-0.7.9"; - /* src = fetchurl { - url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.1.tar.gz; - hash = "sha512-1daoAQfYD0LdhK6aFhrSQvadjc5GsSPBZw0fJDb+BEHYMBLjqiUl2A7H8N+l0W4YfGKqbsPYSrCy4vct+7U6FQ=="; + url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz; + hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ=="; }; - */ - src = lowdown-src; + #src = lowdown-src; outputs = [ "out" "bin" "dev" ]; From 6af6e41df06f0a8a3b919b4052b41d09f0a97678 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 26 Jan 2021 06:22:24 -0500 Subject: [PATCH 079/725] Move command plugin interface to libnixcmd --- Makefile | 1 + src/build-remote/build-remote.cc | 2 +- src/{nix => libcmd}/command.cc | 0 src/{nix => libcmd}/command.hh | 0 src/{nix => libcmd}/installables.cc | 0 src/{nix => libcmd}/installables.hh | 0 src/{nix => libcmd}/legacy.cc | 0 src/{nix => libcmd}/legacy.hh | 0 src/libcmd/local.mk | 15 +++++++++++++++ src/{nix => libcmd}/markdown.cc | 0 src/{nix => libcmd}/markdown.hh | 0 src/libcmd/nix-cmd.pc.in | 9 +++++++++ src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-copy-closure/nix-copy-closure.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/daemon.cc | 2 +- src/nix/local.mk | 4 ++-- 21 files changed, 36 insertions(+), 11 deletions(-) rename src/{nix => libcmd}/command.cc (100%) rename src/{nix => libcmd}/command.hh (100%) rename src/{nix => libcmd}/installables.cc (100%) rename src/{nix => libcmd}/installables.hh (100%) rename src/{nix => libcmd}/legacy.cc (100%) rename src/{nix => libcmd}/legacy.hh (100%) create mode 100644 src/libcmd/local.mk rename src/{nix => libcmd}/markdown.cc (100%) rename src/{nix => libcmd}/markdown.hh (100%) create mode 100644 src/libcmd/nix-cmd.pc.in diff --git a/Makefile b/Makefile index f80b8bb82..68ec3ab0c 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ makefiles = \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ + src/libcmd/local.mk \ src/nix/local.mk \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 17a0a8373..5b8ab3387 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,7 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; using std::cin; diff --git a/src/nix/command.cc b/src/libcmd/command.cc similarity index 100% rename from src/nix/command.cc rename to src/libcmd/command.cc diff --git a/src/nix/command.hh b/src/libcmd/command.hh similarity index 100% rename from src/nix/command.hh rename to src/libcmd/command.hh diff --git a/src/nix/installables.cc b/src/libcmd/installables.cc similarity index 100% rename from src/nix/installables.cc rename to src/libcmd/installables.cc diff --git a/src/nix/installables.hh b/src/libcmd/installables.hh similarity index 100% rename from src/nix/installables.hh rename to src/libcmd/installables.hh diff --git a/src/nix/legacy.cc b/src/libcmd/legacy.cc similarity index 100% rename from src/nix/legacy.cc rename to src/libcmd/legacy.cc diff --git a/src/nix/legacy.hh b/src/libcmd/legacy.hh similarity index 100% rename from src/nix/legacy.hh rename to src/libcmd/legacy.hh diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk new file mode 100644 index 000000000..ab0e0e43d --- /dev/null +++ b/src/libcmd/local.mk @@ -0,0 +1,15 @@ +libraries += libcmd + +libcmd_NAME = libnixcmd + +libcmd_DIR := $(d) + +libcmd_SOURCES := $(wildcard $(d)/*.cc) + +libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers + +libcmd_LDFLAGS = -llowdown + +libcmd_LIBS = libstore libutil libexpr libmain libfetchers + +$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644)) diff --git a/src/nix/markdown.cc b/src/libcmd/markdown.cc similarity index 100% rename from src/nix/markdown.cc rename to src/libcmd/markdown.cc diff --git a/src/nix/markdown.hh b/src/libcmd/markdown.hh similarity index 100% rename from src/nix/markdown.hh rename to src/libcmd/markdown.hh diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in new file mode 100644 index 000000000..1761a9f41 --- /dev/null +++ b/src/libcmd/nix-cmd.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixcmd +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d1c14596c..361f9730d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -17,7 +17,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; using namespace std::string_literals; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 309970df6..57189d557 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include "fetchers.hh" #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 57092b887..c1769790a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,7 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 10990f7b5..ad2e06067 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "store-api.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" using namespace nix; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d6a16999f..106a78fc4 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -14,7 +14,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3956fef6d..ea2e85eb0 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -10,7 +10,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b7eda5ba6..37191b9e6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "graphml.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include #include diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a358cb0d9..26006167d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -8,7 +8,7 @@ #include "globals.hh" #include "derivations.hh" #include "finally.hh" -#include "../nix/legacy.hh" +#include "legacy.hh" #include "daemon.hh" #include diff --git a/src/nix/local.mk b/src/nix/local.mk index 23c08fc86..83b6dd08b 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -14,9 +14,9 @@ nix_SOURCES := \ $(wildcard src/nix-instantiate/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -nix_LIBS = libexpr libmain libfetchers libstore libutil +nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown From d3c428413367a87ab2d27abe9c7f3c379eb12e1c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 5 Jan 2021 10:01:22 +0100 Subject: [PATCH 080/725] Make the error message for missing outputs more useful Don't only show the name of the output, but also the derivation to which this output belongs (as otherwise it's very hard to track back what went wrong) --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01e2fcc7b..9da415c42 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -394,7 +394,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' has no store path mapped to it", outName); + throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); result.insert_or_assign(outName, *optOutPath); } return result; From 9da11bac5797c34b7bb2ee99275befe9c9fb6dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 7 Jan 2021 11:21:43 +0100 Subject: [PATCH 081/725] Fix the error message when a dep is missing Fix a mismatch in the errors thrown when a needed output was missing from an input derivation that was leading to a wrong and quite misleading error message --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..656f92cee 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -539,12 +539,12 @@ void DerivationGoal::inputsRealised() if (!optRealizedInput) throw Error( "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); worker.store.computeFSClosure(*optRealizedInput, inputPaths); } else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); } } } From 8e758d402ba1045c7b8273f8cb1d6d8d917ca52b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 12:06:03 +0100 Subject: [PATCH 082/725] Remove mkFlag() --- src/libmain/shared.cc | 14 ++++++++++---- src/libutil/args.hh | 21 --------------------- src/nix/eval.cc | 6 +++++- src/nix/hash.cc | 43 +++++++++++++++++++++++++++++++++---------- src/nix/ls.cc | 23 ++++++++++++++++++++--- src/nix/path-info.cc | 30 ++++++++++++++++++++++++++---- src/nix/verify.cc | 13 +++++++++++-- 7 files changed, 105 insertions(+), 45 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7e27e95c2..5baaff3e9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -229,11 +229,17 @@ LegacyArgs::LegacyArgs(const std::string & programName, intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "Do not write to the Nix store.", - &settings.readOnlyMode); + addFlag({ + .longName = "readonly-mode", + .description = "Do not write to the Nix store.", + .handler = {&settings.readOnlyMode, true}, + }); - mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", - &gcWarning, false); + addFlag({ + .longName = "no-gc-warning", + .description = "Disable warnings about not using `--add-root`.", + .handler = {&gcWarning, true}, + }); addFlag({ .longName = "store", diff --git a/src/libutil/args.hh b/src/libutil/args.hh index b1020b101..42d8515ef 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -135,27 +135,6 @@ public: void addFlag(Flag && flag); - /* Helper functions for constructing flags / positional - arguments. */ - - void mkFlag(char shortName, const std::string & name, - const std::string & description, bool * dest) - { - mkFlag(shortName, name, description, dest, true); - } - - template - void mkFlag(char shortName, const std::string & longName, const std::string & description, - T * dest, const T & value) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .handler = {[=]() { *dest = value; }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b5049ac65..65d61e005 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,7 +18,11 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); + addFlag({ + .longName = "raw", + .description = "Print strings without quotes or escaping.", + .handler = {&raw, true}, + }); addFlag({ .longName = "apply", diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 79d506ace..4535e4ab0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,18 +19,41 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); - mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); - mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); - mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); + addFlag({ + .longName = "sri", + .description = "Print the hash in SRI format.", + .handler = {&base, SRI}, + }); + + addFlag({ + .longName = "base64", + .description = "Print the hash in base-64 format.", + .handler = {&base, Base64}, + }); + + addFlag({ + .longName = "base32", + .description = "Print the hash in base-32 (Nix-specific) format.", + .handler = {&base, Base32}, + }); + + addFlag({ + .longName = "base16", + .description = "Print the hash in base-16 format.", + .handler = {&base, Base16}, + }); + addFlag(Flag::mkHashTypeFlag("type", &ht)); + #if 0 - mkFlag() - .longName("modulo") - .description("Compute the hash modulo specified the string.") - .labels({"modulus"}) - .dest(&modulus); - #endif + addFlag({ + .longName = "modulo", + .description = "Compute the hash modulo the specified string.", + .labels = {"modulus"}, + .handler = {&modulus}, + }); + #endif\ + expectArgs({ .label = "paths", .handler = {&paths}, diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c0b1ecb32..c1dc9a95b 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,26 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); - mkFlag('l', "long", "Show detailed file information.", &verbose); - mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); + addFlag({ + .longName = "recursive", + .shortName = 'R', + .description = "List subdirectories recursively.", + .handler = {&recursive, true}, + }); + + addFlag({ + .longName = "long", + .shortName = 'l', + .description = "Show detailed file information.", + .handler = {&verbose, true}, + }); + + addFlag({ + .longName = "directory", + .shortName = 'd', + .description = "Show directories rather than their contents.", + .handler = {&showDirectory, true}, + }); } void listText(ref accessor) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0fa88f1bf..518cd5568 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); - mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); - mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); - mkFlag(0, "sigs", "Show signatures.", &showSigs); + addFlag({ + .longName = "size", + .shortName = 's', + .description = "Print the size of the NAR serialisation of each path.", + .handler = {&showSize, true}, + }); + + addFlag({ + .longName = "closure-size", + .shortName = 'S', + .description = "Print the sum of the sizes of the NAR serialisations of the closure of each path.", + .handler = {&showClosureSize, true}, + }); + + addFlag({ + .longName = "human-readable", + .shortName = 'h', + .description = "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", + .handler = {&humanReadable, true}, + }); + + addFlag({ + .longName = "sigs", + .description = "Show signatures.", + .handler = {&showSigs, true}, + }); } std::string description() override diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b04e032a..1721c7f16 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,8 +18,17 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); - mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); + addFlag({ + .longName = "no-contents", + .description = "Do not verify the contents of each store path.", + .handler = {&noContents, true}, + }); + + addFlag({ + .longName = "no-trust", + .description = "Do not verify whether each store path is trusted.", + .handler = {&noTrust, true}, + }); addFlag({ .longName = "substituter", From c03f41055de6f885ade7fa7927bf83fb697a3dba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:02:54 +0100 Subject: [PATCH 083/725] Add traces to errors while updating flake lock file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example: $ nix build --show-trace error: unable to download 'https://api.github.com/repos/NixOS/nixpkgs/commits/no-such-branch': HTTP error 422 ('') response body: { "message": "No commit found for SHA: no-such-branch", "documentation_url": "https://docs.github.com/rest/reference/repos#get-a-commit" } … while fetching the input 'github:NixOS/nixpkgs/no-such-branch' … while updating the flake input 'nixpkgs' … while updating the lock file of flake 'git+file:///home/eelco/Dev/nix' --- src/libexpr/flake/flake.cc | 474 +++++++++++++++++++----------------- src/libfetchers/fetchers.cc | 9 +- 2 files changed, 252 insertions(+), 231 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0786fef3d..2e94490d4 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -298,284 +298,298 @@ LockedFlake lockFlake( auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); - // FIXME: symlink attack - auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + try { - debug("old lock file: %s", oldLockFile); + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); - // FIXME: check whether all overrides are used. - std::map overrides; - std::set overridesUsed, updatesUsed; + debug("old lock file: %s", oldLockFile); - for (auto & i : lockFlags.inputOverrides) - overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + // FIXME: check whether all overrides are used. + std::map overrides; + std::set overridesUsed, updatesUsed; - LockFile newLockFile; + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); - std::vector parents; + LockFile newLockFile; - std::function node, - const InputPath & inputPathPrefix, - std::shared_ptr oldNode)> - computeLocks; + std::vector parents; - computeLocks = [&]( - const FlakeInputs & flakeInputs, - std::shared_ptr node, - const InputPath & inputPathPrefix, - std::shared_ptr oldNode) - { - debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + std::function node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode)> + computeLocks; - /* Get the overrides (i.e. attributes of the form - 'inputs.nixops.inputs.nixpkgs.url = ...'). */ - // FIXME: check this - for (auto & [id, input] : flake.inputs) { - for (auto & [idOverride, inputOverride] : input.overrides) { + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr node, + const InputPath & inputPathPrefix, + std::shared_ptr oldNode) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { auto inputPath(inputPathPrefix); inputPath.push_back(id); - inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); - } - } + auto inputPathS = printInputPath(inputPath); + debug("computing input '%s'", inputPathS); - /* Go over the flake inputs, resolve/fetch them if - necessary (i.e. if they're new or the flakeref changed - from what's in the lock file). */ - for (auto & [id, input2] : flakeInputs) { - auto inputPath(inputPathPrefix); - inputPath.push_back(id); - auto inputPathS = printInputPath(inputPath); - debug("computing input '%s'", inputPathS); + try { - /* Do we have an override for this input from one of the - ancestors? */ - auto i = overrides.find(inputPath); - bool hasOverride = i != overrides.end(); - if (hasOverride) overridesUsed.insert(inputPath); - auto & input = hasOverride ? i->second : input2; + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; - /* Resolve 'follows' later (since it may refer to an input - path we haven't processed yet. */ - if (input.follows) { - InputPath target; - if (hasOverride || input.absolute) - /* 'follows' from an override is relative to the - root of the graph. */ - target = *input.follows; - else { - /* Otherwise, it's relative to the current flake. */ - target = inputPathPrefix; - for (auto & i : *input.follows) target.push_back(i); - } - debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); - node->inputs.insert_or_assign(id, target); - continue; - } + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputPath target; + if (hasOverride || input.absolute) + /* 'follows' from an override is relative to the + root of the graph. */ + target = *input.follows; + else { + /* Otherwise, it's relative to the current flake. */ + target = inputPathPrefix; + for (auto & i : *input.follows) target.push_back(i); + } + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } - assert(input.ref); + assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ - std::shared_ptr oldLock; + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr oldLock; - updatesUsed.insert(inputPath); + updatesUsed.insert(inputPath); - if (oldNode && !lockFlags.inputUpdates.count(inputPath)) - if (auto oldLock2 = get(oldNode->inputs, id)) - if (auto oldLock3 = std::get_if<0>(&*oldLock2)) - oldLock = *oldLock3; + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; - if (oldLock - && oldLock->originalRef == *input.ref - && !hasOverride) - { - debug("keeping existing input '%s'", inputPathS); + if (oldLock + && oldLock->originalRef == *input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); - /* Copy the input from the old lock since its flakeref - didn't change and there is no override from a - higher level flake. */ - auto childNode = std::make_shared( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared( + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); - node->inputs.insert_or_assign(id, childNode); + node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = - lb != lockFlags.inputUpdates.end() - && lb->size() > inputPath.size() - && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); - } else { - /* No need to fetch this flake, we can be - lazy. However there may be new overrides on the - inputs of this flake, so we need to check - those. */ - FlakeInputs fakeInputs; + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { - if (auto lockedNode = std::get_if<0>(&i.second)) { - fakeInputs.emplace(i.first, FlakeInput { - .ref = (*lockedNode)->originalRef, - .isFlake = (*lockedNode)->isFlake, - }); - } else if (auto follows = std::get_if<1>(&i.second)) { - fakeInputs.emplace(i.first, FlakeInput { - .follows = *follows, - .absolute = true - }); + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .follows = *follows, + .absolute = true + }); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputPathS); + + if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared( + inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared(lockedRef, *input.ref, false)); } } - computeLocks(fakeInputs, childNode, inputPath, oldLock); - } - - } else { - /* We need to create a new lock file entry. So fetch - this input. */ - debug("creating new input '%s'", inputPathS); - - if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) - throw Error("cannot update flake input '%s' in pure mode", inputPathS); - - if (input.isFlake) { - auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache); - - /* Note: in case of an --override-input, we use - the *original* ref (input2.ref) for the - "original" field, rather than the - override. This ensures that the override isn't - nuked the next time we update the lock - file. That is, overrides are sticky unless you - use --no-write-lock-file. */ - auto childNode = std::make_shared( - inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); - - node->inputs.insert_or_assign(id, childNode); - - /* Guard against circular flake imports. */ - for (auto & parent : parents) - if (parent == *input.ref) - throw Error("found circular import of flake '%s'", parent); - parents.push_back(*input.ref); - Finally cleanup([&]() { parents.pop_back(); }); - - /* Recursively process the inputs of this - flake. Also, unless we already have this flake - in the top-level lock file, use this flake's - own lock file. */ - computeLocks( - inputFlake.inputs, childNode, inputPath, - oldLock - ? std::dynamic_pointer_cast(oldLock) - : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); - } - - else { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, lockFlags.useRegistries, flakeCache); - node->inputs.insert_or_assign(id, - std::make_shared(lockedRef, *input.ref, false)); + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", inputPathS); + throw; } } - } - }; + }; - computeLocks( - flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); - for (auto & i : lockFlags.inputOverrides) - if (!overridesUsed.count(i.first)) - warn("the flag '--override-input %s %s' does not match any input", - printInputPath(i.first), i.second); + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + printInputPath(i.first), i.second); - for (auto & i : lockFlags.inputUpdates) - if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", printInputPath(i)); - /* Check 'follows' inputs. */ - newLockFile.check(); + /* Check 'follows' inputs. */ + newLockFile.check(); - debug("new lock file: %s", newLockFile); + debug("new lock file: %s", newLockFile); - /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { - auto diff = LockFile::diff(oldLockFile, newLockFile); + auto diff = LockFile::diff(oldLockFile, newLockFile); - if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { - if (!newLockFile.isImmutable()) { - if (settings.warnDirty) - warn("will not write lock file of flake '%s' because it has a mutable input", topRef); - } else { - if (!lockFlags.updateLockFile) - throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input.getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto path = *sourcePath + "/" + relPath; + auto path = *sourcePath + "/" + relPath; - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(path); - if (lockFileExists) { - auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", path); - else - warn("updating lock file '%s':\n%s", path, s); - } else - warn("creating lock file '%s'", path); + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", path); + else + warn("updating lock file '%s':\n%s", path, s); + } else + warn("creating lock file '%s'", path); - newLockFile.write(path); + newLockFile.write(path); - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - lockFlags.commitLockFile - ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", - relPath, lockFileExists ? "Update" : "Add", diff)) - : std::nullopt); + topRef.input.markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); - /* Rewriting the lockfile changed the top-level - repo, so we should re-read it. FIXME: we could - also just clear the 'rev' field... */ - auto prevLockedRef = flake.lockedRef; - FlakeCache dummyCache; - flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); - if (lockFlags.commitLockFile && - flake.lockedRef.input.getRev() && - prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) - warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + if (lockFlags.commitLockFile && + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); - /* Make sure that we picked up the change, - i.e. the tree should usually be dirty - now. Corner case: we could have reverted from a - dirty to a clean tree! */ - if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input.isImmutable()) - throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); - } + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input.isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); } else - throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); - } else - warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); - } + warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + } - return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + + } catch (Error & e) { + e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); + throw; + } } void callFlake(EvalState & state, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e6741a451..916e0a8e8 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -132,7 +132,14 @@ std::pair Input::fetch(ref store) const } } - auto [tree, input] = scheme->fetch(store, *this); + auto [tree, input] = [&]() -> std::pair { + try { + return scheme->fetch(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } + }(); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); From 965dc6070a1b7dc582d90039c670d436f4a2e9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:04:49 +0100 Subject: [PATCH 084/725] Drop trailing whitespace --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 563f49170..8ea5cdc9d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -856,7 +856,7 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); else err.msg = hf; } From 12de0466fea6558ccb0dd5b98b72d7a068c9b5e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:46:10 +0100 Subject: [PATCH 085/725] Add trace to build errors during import-from-derivation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example: error: builder for '/nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv' failed with exit code 1; last 1 log lines: > FAIL For full logs, run 'nix log /nix/store/9ysqfidhipyzfiy54mh77iqn29j6cpsb-failing.drv'. … while importing '/nix/store/pfp4a4bjh642ylxyipncqs03z6kkgfvy-failing' at /nix/store/25wgzr2qrqqiqfbdb1chpiry221cjglc-source/flake.nix:58:15: 57| 58| ifd = import self.hydraJobs.broken; | ^ 59| --- src/libexpr/primops.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a470ed6df..13565b950 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -118,6 +118,9 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); + } catch (Error & e) { + e.addTrace(pos, "while importing '%s'", path); + throw e; } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); From 9355ecd54301372b6a919a2205340f904c7a51c6 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 17:24:30 +0100 Subject: [PATCH 086/725] Add a new Cmd type working on RealisedPaths Where a `RealisedPath` is a store path with its history, meaning either an opaque path for stuff that has been directly added to the store, or a `Realisation` for stuff that has been built by a derivation This is a low-level refactoring that doesn't bring anything by itself (except a few dozen extra lines of code :/ ), but raising the abstraction level a bit is important on a number of levels: - Commands like `nix build` have to query for the realisations after the build is finished which is fragile (see 27905f12e4a7207450abe37c9ed78e31603b67e1 for example). Having them oprate directly at the realisation level would avoid that - Others like `nix copy` currently operate directly on (built) store paths, but need a bit more information as they will need to register the realisations on the remote side --- src/libcmd/command.cc | 42 ++++++++++------- src/libcmd/command.hh | 23 ++++++++-- src/libcmd/installables.cc | 48 ++++++++++++++++---- src/libstore/realisation.cc | 31 +++++++++++++ src/libstore/realisation.hh | 90 +++++++++++++++++++++++++++++++++---- src/nix/copy.cc | 2 + 6 files changed, 200 insertions(+), 36 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 614dee788..efdc98d5a 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,7 +54,7 @@ void StoreCommand::run() run(getStore()); } -StorePathsCommand::StorePathsCommand(bool recursive) +RealisedPathsCommand::RealisedPathsCommand(bool recursive) : recursive(recursive) { if (recursive) @@ -81,30 +81,40 @@ StorePathsCommand::StorePathsCommand(bool recursive) }); } -void StorePathsCommand::run(ref store) +void RealisedPathsCommand::run(ref store) { - StorePaths storePaths; - + std::vector paths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); + // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); - } - - else { - for (auto & p : toStorePaths(store, realiseMode, operateOn, installables)) - storePaths.push_back(p); - + paths.push_back(p); + } else { + auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables); if (recursive) { - StorePathSet closure; - store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); - storePaths.clear(); - for (auto & p : closure) - storePaths.push_back(p); + auto roots = std::move(pathSet); + pathSet = {}; + RealisedPath::closure(*store, roots, pathSet); } + for (auto & path : pathSet) + paths.push_back(path); } + run(store, std::move(paths)); +} + +StorePathsCommand::StorePathsCommand(bool recursive) + : RealisedPathsCommand(recursive) +{ +} + +void StorePathsCommand::run(ref store, std::vector paths) +{ + StorePaths storePaths; + for (auto & p : paths) + storePaths.push_back(p.path()); + run(store, std::move(storePaths)); } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index ed6980075..8c0b3a94a 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -141,7 +141,7 @@ private: }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public InstallablesCommand +struct RealisedPathsCommand : public InstallablesCommand { private: @@ -154,17 +154,28 @@ protected: public: - StorePathsCommand(bool recursive = false); + RealisedPathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, std::vector storePaths) = 0; + virtual void run(ref store, std::vector paths) = 0; void run(ref store) override; bool useDefaultInstallables() override { return !all; } }; +struct StorePathsCommand : public RealisedPathsCommand +{ + StorePathsCommand(bool recursive = false); + + using RealisedPathsCommand::run; + + virtual void run(ref store, std::vector storePaths) = 0; + + void run(ref store, std::vector paths) override; +}; + /* A command that operates on exactly one store path. */ struct StorePathCommand : public InstallablesCommand { @@ -218,6 +229,12 @@ std::set toDerivations(ref store, std::vector> installables, bool useDeriver = false); +std::set toRealisedPaths( + ref store, + Realise mode, + OperateOn operateOn, + std::vector> installables); + /* Helper function to generate args that invoke $EDITOR on filename:lineno. */ Strings editorFor(const Pos & pos); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4e6bf4a9a..98a27ded9 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -704,23 +704,43 @@ Buildables build(ref store, Realise mode, return buildables; } -StorePathSet toStorePaths(ref store, - Realise mode, OperateOn operateOn, +std::set toRealisedPaths( + ref store, + Realise mode, + OperateOn operateOn, std::vector> installables) { - StorePathSet outPaths; - + std::set res; if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { [&](BuildableOpaque bo) { - outPaths.insert(bo.path); + res.insert(bo.path); }, [&](BuildableFromDrv bfd) { + auto drv = store->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { - if (!output.second) - throw Error("Cannot operate on output of unbuilt CA drv"); - outPaths.insert(*output.second); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (!outputHashes.count(output.first)) + throw Error( + "The derivation %s doesn't have an output " + "named %s", + store->printStorePath(bfd.drvPath), + output.first); + auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string()); + res.insert(RealisedPath{*realisation}); + } + else { + // If ca-derivations isn't enabled, behave as if + // all the paths are opaque to keep the default + // behavior + assert(output.second); + res.insert(*output.second); + } } }, }, b); @@ -731,9 +751,19 @@ StorePathSet toStorePaths(ref store, for (auto & i : installables) for (auto & b : i->toBuildables()) if (auto bfd = std::get_if(&b)) - outPaths.insert(bfd->drvPath); + res.insert(bfd->drvPath); } + return res; +} + +StorePathSet toStorePaths(ref store, + Realise mode, OperateOn operateOn, + std::vector> installables) +{ + StorePathSet outPaths; + for (auto & path : toRealisedPaths(store, mode, operateOn, installables)) + outPaths.insert(path.path()); return outPaths; } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47ad90eee..c9b66186f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -46,4 +46,35 @@ Realisation Realisation::fromJSON( }; } +StorePath RealisedPath::path() const { + return visit([](auto && arg) { return arg.getPath(); }); +} + +void RealisedPath::closure( + Store& store, + const RealisedPath::Set& startPaths, + RealisedPath::Set& ret) +{ + // FIXME: This only builds the store-path closure, not the real realisation + // closure + StorePathSet initialStorePaths, pathsClosure; + for (auto& path : startPaths) + initialStorePaths.insert(path.path()); + store.computeFSClosure(initialStorePaths, pathsClosure); + ret.insert(startPaths.begin(), startPaths.end()); + ret.insert(pathsClosure.begin(), pathsClosure.end()); +} + +void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +{ + RealisedPath::closure(store, {*this}, ret); +} + +RealisedPath::Set RealisedPath::closure(Store& store) const +{ + RealisedPath::Set ret; + closure(store, ret); + return ret; +} + } // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 4b8ead3c5..1ecddc4d1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,34 @@ #include "path.hh" #include + +/* Awfull hacky generation of the comparison operators by doing a lexicographic + * comparison between the choosen fields + * ``` + * GENERATE_CMP(ClassName, my->field1, my->field2, ...) + * ``` + * + * will generate comparison operators semantically equivalent to: + * ``` + * bool operator<(const ClassName& other) { + * return field1 < other.field1 && field2 < other.field2 && ...; + * } + * ``` + */ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ + bool operator COMPARATOR(const MY_TYPE& other) const { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple( FIELDS ); \ + me = &other; \ + auto fields2 = std::make_tuple( FIELDS ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) +#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_CMP(args...) \ + GENERATE_EQUAL(args) \ + GENERATE_LEQ(args) + namespace nix { struct DrvOutput { @@ -17,13 +45,7 @@ struct DrvOutput { static DrvOutput parse(const std::string &); - bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } - bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } - -private: - // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvHash, outputName); } + GENERATE_CMP(DrvOutput, me->drvHash, me->outputName); }; struct Realisation { @@ -32,8 +54,60 @@ struct Realisation { nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + + StorePath getPath() const { return outPath; } + + GENERATE_CMP(Realisation, me->id, me->outPath); }; -typedef std::map DrvOutputs; +struct OpaquePath { + StorePath path; + + StorePath getPath() const { return path; } + + GENERATE_CMP(OpaquePath, me->path); +}; + + +/** + * A store path with all the history of how it went into the store + */ +struct RealisedPath { + /* + * A path is either the result of the realisation of a derivation or + * an opaque blob that has been directly added to the store + */ + using Raw = std::variant; + Raw raw; + + using Set = std::set; + + RealisedPath(StorePath path) : raw(OpaquePath{path}) {} + RealisedPath(Realisation r) : raw(r) {} + + /** + * Syntactic sugar to run `std::visit` on the raw value: + * path.visit(blah) == std::visit(blah, path.raw) + */ + template + constexpr decltype(auto) visit(Visitor && vis) { + return std::visit(vis, raw); + } + template + constexpr decltype(auto) visit(Visitor && vis) const { + return std::visit(vis, raw); + } + + /** + * Get the raw store path associated to this + */ + StorePath path() const; + + void closure(Store& store, Set& ret) const; + static void closure(Store& store, const Set& startPaths, Set& ret); + Set closure(Store& store) const; + + GENERATE_CMP(RealisedPath, me->raw); +}; } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f15031a45..c56a1def1 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -16,6 +16,8 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; + using StorePathsCommand::run; + CmdCopy() : StorePathsCommand(true) { From 991edaace57d50d571f4f4658ef2d52b94a07f2c Mon Sep 17 00:00:00 2001 From: James Ottaway Date: Fri, 29 Jan 2021 13:55:18 +1000 Subject: [PATCH 087/725] Shorten `mktemp` flag for macOS Address `mktemp: illegal option -- -`. --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 578258394..3c44fdb0e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -239,7 +239,7 @@ struct Common : InstallableCommand, MixProfile out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); From d5acc4865c8a5853bc5ede606d98c8055f8afdb2 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 29 Jan 2021 18:31:40 +0100 Subject: [PATCH 088/725] Use passthru for perl-bindings, allows Nix patching for Hydra This allows patching Nix for Hydra with additional overlays, because `.overrideAttrs` and co. will persist the passthru's --- flake.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 9addccd63..830cceb9f 100644 --- a/flake.nix +++ b/flake.nix @@ -115,7 +115,7 @@ # 'nix.perl-bindings' packages. overlay = final: prev: { - nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -163,9 +163,8 @@ installCheckFlags = "sysconfdir=$(out)/etc"; separateDebugInfo = true; - }) // { - perl-bindings = with final; stdenv.mkDerivation { + passthru.perl-bindings = with final; stdenv.mkDerivation { name = "nix-perl-${version}"; src = self; From d0b74e2d2506b9237263ad1294eb7297c99a5e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 1 Feb 2021 13:11:42 +0000 Subject: [PATCH 089/725] --no-net -> --offline --- src/nix/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 58b643cc5..e95b04d85 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -91,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs }); addFlag({ - .longName = "no-net", + .longName = "offline", .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); From fb00e7dc529f54e6b2d864532e93ef3645b1b704 Mon Sep 17 00:00:00 2001 From: Dominik Schrempf Date: Mon, 1 Feb 2021 17:42:14 +0100 Subject: [PATCH 090/725] Remove newline in operator table. --- doc/manual/src/expressions/language-operators.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index 1d787ffe3..b7fd6f4c6 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -25,5 +25,4 @@ order of precedence (from strongest to weakest binding). | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | | Logical OR | *e1* `\|\|` *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| - e2`). | 14 | +| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| e2`). | 14 | From 3d1bbabe55eff6e67d91e0cbee781c2b756a2e92 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 2 Feb 2021 19:50:03 -0600 Subject: [PATCH 091/725] Use derivation output name from toDerivation This fixes an issue where derivations with a primary output that is not "out" would fail with: $ nix profile install nixpkgs#sqlite error: opening directory '/nix/store/2a2ydlgyydly5czcc8lg12n6qqkfz863-sqlite-3.34.1-bin': No such file or directory This happens because while derivations produce every output when built, you might not have them if you didn't build the derivation yourself (for instance, the store path was fetch from a binary cache). This uses outputName provided from DerivationInfo which appears to match the first output of the derivation. --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 765d6866e..827f8be5a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { From 76d8bdfe355aa1976580f4fa8f11f1ec505a6c66 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 2 Feb 2021 23:04:36 +0100 Subject: [PATCH 092/725] Include note about type of catched errors in tryEval documentation Reference #356. --- src/libexpr/primops.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 13565b950..1d1afa768 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -696,10 +696,14 @@ static RegisterPrimOp primop_tryEval({ Try to shallowly evaluate *e*. Return a set containing the attributes `success` (`true` if *e* evaluated successfully, `false` if an error was thrown) and `value`, equalling *e* if - successful and `false` otherwise. Note that this doesn't evaluate - *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval - e).success ` will be `true`. Using ` builtins.deepSeq ` one can - get the expected result: `let e = { x = throw ""; }; in + successful and `false` otherwise. `tryEval` will only prevent + errors created by `throw` or `assert` from being thrown. + Errors `tryEval` will not catch are for example those created + by `abort` and type errors generated by builtins. Also note that + this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; + in (builtins.tryEval e).success` will be `true`. Using + `builtins.deepSeq` one can get the expected result: + `let e = { x = throw ""; }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be `false`. )", From e38cd5becbbff57951b6a576dd793f4777a9833c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Feb 2021 21:22:11 -0600 Subject: [PATCH 093/725] Always enter first level of attrset in nix search This makes nix search always go through the first level of an attribute set, even if it's not a top level attribute. For instance, you can now list all GHC compilers with: $ nix search nixpkgs#haskell.compiler ... This is similar to how nix-env works when you pass in -A. --- src/nix/search.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 9f864b3a4..c52a48d4e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -81,9 +81,9 @@ struct CmdSearch : InstallableCommand, MixJSON uint64_t results = 0; - std::function & attrPath)> visit; + std::function & attrPath, bool initialRecurse)> visit; - visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath) + visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath, bool initialRecurse) { Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPath))); @@ -94,7 +94,7 @@ struct CmdSearch : InstallableCommand, MixJSON auto cursor2 = cursor.getAttr(attr); auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*cursor2, attrPath2); + visit(*cursor2, attrPath2, false); } }; @@ -150,6 +150,9 @@ struct CmdSearch : InstallableCommand, MixJSON || (attrPath[0] == "packages" && attrPath.size() <= 2)) recurse(); + else if (initialRecurse) + recurse(); + else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); if (attr && attr->getBool()) @@ -163,7 +166,7 @@ struct CmdSearch : InstallableCommand, MixJSON }; for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix)); + visit(*cursor, parseAttrPath(*state, prefix), true); if (!json && !results) throw Error("no results for the given search term(s)!"); From ca8facefb6b6b0ffd6e22507111847dbfc9a3c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 14:47:28 +0100 Subject: [PATCH 094/725] Normalize some error messages Co-authored-by: Eelco Dolstra --- src/libcmd/installables.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 98a27ded9..9ad02b5f0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -724,14 +724,13 @@ std::set toRealisedPaths( if (settings.isExperimentalFeatureEnabled("ca-derivations")) { if (!outputHashes.count(output.first)) throw Error( - "The derivation %s doesn't have an output " - "named %s", + "the derivation '%s' doesn't have an output named '%s'", store->printStorePath(bfd.drvPath), output.first); auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; auto realisation = store->queryRealisation(outputId); if (!realisation) - throw Error("Cannot operate on output of unbuilt CA drv %s", outputId.to_string()); + throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string()); res.insert(RealisedPath{*realisation}); } else { From 43d409f6690b79b5d4e1ab5e9780de93eb0f677a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 14:47:56 +0100 Subject: [PATCH 095/725] Fix a whitespace issue Co-authored-by: Eelco Dolstra --- src/libstore/realisation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index c9b66186f..e4276c040 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -65,7 +65,7 @@ void RealisedPath::closure( ret.insert(pathsClosure.begin(), pathsClosure.end()); } -void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const { RealisedPath::closure(store, {*this}, ret); } From d2091af231ab97b729c2486b55e520c565e59dd3 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:11:05 +0100 Subject: [PATCH 096/725] Move the GENERATE_CMP macro to its own file Despite being an ugly hack, it can probably be useful in a couple extra places --- src/libstore/realisation.hh | 29 +---------------------------- src/libutil/comparator.hh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 src/libutil/comparator.hh diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 1ecddc4d1..557f54362 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -2,34 +2,7 @@ #include "path.hh" #include - - -/* Awfull hacky generation of the comparison operators by doing a lexicographic - * comparison between the choosen fields - * ``` - * GENERATE_CMP(ClassName, my->field1, my->field2, ...) - * ``` - * - * will generate comparison operators semantically equivalent to: - * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; - * } - * ``` - */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) +#include "comparator.hh" namespace nix { diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh new file mode 100644 index 000000000..0315dc506 --- /dev/null +++ b/src/libutil/comparator.hh @@ -0,0 +1,30 @@ +#pragma once + +/* Awfull hacky generation of the comparison operators by doing a lexicographic + * comparison between the choosen fields. + * + * ``` + * GENERATE_CMP(ClassName, me->field1, me->field2, ...) + * ``` + * + * will generate comparison operators semantically equivalent to: + * + * ``` + * bool operator<(const ClassName& other) { + * return field1 < other.field1 && field2 < other.field2 && ...; + * } + * ``` + */ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ + bool operator COMPARATOR(const MY_TYPE& other) const { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple( FIELDS ); \ + me = &other; \ + auto fields2 = std::make_tuple( FIELDS ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) +#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_CMP(args...) \ + GENERATE_EQUAL(args) \ + GENERATE_LEQ(args) From e69cfdebb090b3aabbff69a44504883d5b6fb866 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:15:22 +0100 Subject: [PATCH 097/725] Remove the `visit` machinery in `RealisedPath` In addition to being some ugly template trickery, it was also totally useless as it was used in only one place where I could replace it by just a few extra characters --- src/libstore/realisation.cc | 2 +- src/libstore/realisation.hh | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index e4276c040..cd74af4ee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -47,7 +47,7 @@ Realisation Realisation::fromJSON( } StorePath RealisedPath::path() const { - return visit([](auto && arg) { return arg.getPath(); }); + return std::visit([](auto && arg) { return arg.getPath(); }, raw); } void RealisedPath::closure( diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 557f54362..7c91d802a 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -58,19 +58,6 @@ struct RealisedPath { RealisedPath(StorePath path) : raw(OpaquePath{path}) {} RealisedPath(Realisation r) : raw(r) {} - /** - * Syntactic sugar to run `std::visit` on the raw value: - * path.visit(blah) == std::visit(blah, path.raw) - */ - template - constexpr decltype(auto) visit(Visitor && vis) { - return std::visit(vis, raw); - } - template - constexpr decltype(auto) visit(Visitor && vis) const { - return std::visit(vis, raw); - } - /** * Get the raw store path associated to this */ From 0187838e2e7ff01f1b480e3e85d9e96da0b4b78e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 12:11:50 +0100 Subject: [PATCH 098/725] Add a trace to readLine() failures Hopefully this helps to diagnose 'error: unexpected EOF reading a line' on macOS. --- src/libstore/build/derivation-goal.cc | 25 ++++++++++++++++++++++--- tests/init.sh | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8717499c0..190adf31c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1044,7 +1044,14 @@ HookReply DerivationGoal::tryBuildHook() whether the hook wishes to perform the build. */ string reply; while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); + auto s = [&]() { + try { + return readLine(worker.hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response from the build hook"); + throw e; + } + }(); if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) ; else if (string(s, 0, 2) == "# ") { @@ -1084,7 +1091,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); - machineName = readLine(hook->fromHook.readSide.get()); + try { + machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw e; + } /* Tell the hook all the inputs that have to be copied to the remote system. */ @@ -1773,7 +1785,14 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { - string msg = readLine(builderOut.readSide.get()); + string msg = [&]() { + try { + return readLine(builderOut.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response of setting up the build environment"); + throw e; + } + }(); if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { FdSource source(builderOut.readSide.get()); diff --git a/tests/init.sh b/tests/init.sh index 63cf895e2..1a6ccb6fe 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -21,6 +21,7 @@ experimental-features = nix-command flakes gc-reserved-space = 0 substituters = flake-registry = $TEST_ROOT/registry.json +show-trace = true include nix.conf.extra EOF From 480426a364f09e7992230b32f2941a09fb52d729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 15:57:33 +0100 Subject: [PATCH 099/725] Add more instrumentation for #4270 --- src/libstore/build/derivation-goal.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 190adf31c..eeaec4f2c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1784,12 +1784,14 @@ void DerivationGoal::startBuilder() worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); /* Check if setting up the build environment failed. */ + std::vector msgs; while (true) { string msg = [&]() { try { return readLine(builderOut.readSide.get()); } catch (Error & e) { - e.addTrace({}, "while reading the response of setting up the build environment"); + e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + concatStringsSep("|", msgs)); throw e; } }(); @@ -1801,6 +1803,7 @@ void DerivationGoal::startBuilder() throw ex; } debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); } } From 4fefe26717fa70828e3f524e43c76e3f7b7a09b0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 5 Feb 2021 18:22:34 -0600 Subject: [PATCH 100/725] Re-enable armv6l support This fixes the libatomic detection. --- configure.ac | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 2047ed8d2..685c471c5 100644 --- a/configure.ac +++ b/configure.ac @@ -152,7 +152,7 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="$LDFLAGS -latomic" fi PKG_PROG_PKG_CONFIG diff --git a/flake.nix b/flake.nix index 869b92cb7..7e02fd70d 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" ]; - crossSystems = [ "armv7l-linux" ]; + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); From d0e34c85f85510cb2ef591de29693b4cf8bdc65b Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Sat, 6 Feb 2021 12:59:11 +0100 Subject: [PATCH 101/725] libcmd/markdown: handle allocation errors in lowdown_term_rndr We upgrade to lowdown 0.8.0 [1] which contains a fix/improvement to a behavior mentioned in this issue thread [2] where a big part of lowdown's API would just call exit(1) on allocation errors since that is a satisfying behavior for the lowdown binary. Now lowdown_term_rndr returns 0 if an allocation error occurred which we check for in libcmd/markdown.cc. Also the extern "C" { } wrapper around lowdown.h has been removed as it is not necessary. [1]: https://github.com/kristapsdz/lowdown/blob/6ca7c855a063d1c77ae0b89405047cc3913a74d8/versions.xml#L987-L1006 [2]: https://github.com/kristapsdz/lowdown/issues/45#issuecomment-756681153 --- flake.nix | 8 ++++---- src/libcmd/markdown.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index d94da9dae..8c60934e6 100644 --- a/flake.nix +++ b/flake.nix @@ -198,12 +198,12 @@ }; - lowdown = with final; stdenv.mkDerivation { - name = "lowdown-0.7.9"; + lowdown = with final; stdenv.mkDerivation rec { + name = "lowdown-0.8.0"; src = fetchurl { - url = https://kristaps.bsd.lv/lowdown/snapshots/lowdown-0.7.9.tar.gz; - hash = "sha512-7GQrKFICyTI5T4SinATfohiCq9TC0OgN8NmVfG3B3BZJM9J00DT8llAco8kNykLIKtl/AXuS4X8fETiCFEWEUQ=="; + url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz"; + hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw=="; }; #src = lowdown-src; diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 40788a42f..d25113d93 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -3,9 +3,7 @@ #include "finally.hh" #include -extern "C" { #include -} namespace nix { @@ -42,7 +40,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("cannot allocate Markdown output buffer"); Finally freeBuffer([&]() { lowdown_buf_free(buf); }); - lowdown_term_rndr(buf, nullptr, renderer, node); + int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node); + if (!rndr_res) + throw Error("allocation error while rendering Markdown"); return std::string(buf->data, buf->size); } From 6af26b7aec28e8bf1786ead3ba26beb50317c167 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Sat, 6 Feb 2021 13:29:38 +0100 Subject: [PATCH 102/725] Add Stale bot The configuration was taken from nixpkgs repository and adjusted to `NixOS/nix`. A `stale` label was added to the labels (with gray color). Issues and PRs with `critical` label are excluded from interacting with the stale bot. --- .github/STALE-BOT.md | 35 +++++++++++++++++++++++++++++++++++ .github/stale.yml | 9 +++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/STALE-BOT.md create mode 100644 .github/stale.yml diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md new file mode 100644 index 000000000..6cc03f540 --- /dev/null +++ b/.github/STALE-BOT.md @@ -0,0 +1,35 @@ +# Stale bot information + +- Thanks for your contribution! +- To remove the stale label, just leave a new comment. +- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos). + +## Suggestions for PRs + +1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else. +2. If it is unfinished but you plan to finish it, please mark it as a draft. +3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. +4. To get things rolling again, rebase the PR against the target branch and address valid comments. +5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604). +6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions. + +## Suggestions for issues + +1. If it is resolved (either for you personally, or in general), please consider closing it. +2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough. +3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/). +4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude! + +**Memorandum on closing issues** + +Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort. + +## Useful GitHub search queries + +- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) +- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) +- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+) +- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..f81b4c762 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,9 @@ +# Configuration for probot-stale - https://github.com/probot/stale +daysUntilStale: 180 +daysUntilClose: false +exemptLabels: + - "critical" +staleLabel: "2.status: stale" +markComment: | + I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) +closeComment: false From 91d83426f70bbf28c1bf92be5f662d76d1d47578 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Sat, 6 Feb 2021 13:33:34 +0100 Subject: [PATCH 103/725] typo --- .github/STALE-BOT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 6cc03f540..5e8f5d929 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -27,7 +27,7 @@ Don't be afraid to close an issue that holds valuable information. Closed issues ## Useful GitHub search queries -- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nixs/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) +- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) - [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) - [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) - [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) From 7c112351d9e941567e64063638396259546d9a48 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Sun, 7 Feb 2021 13:56:50 +0000 Subject: [PATCH 104/725] libutil: EPERM from kill(-1, ...) is fine I tested a trivial program that called kill(-1, SIGKILL), which was run as the only process for an unpriveleged user, on Linux and FreeBSD. On Linux, kill reported success, while on FreeBSD it failed with EPERM. POSIX says: > If pid is -1, sig shall be sent to all processes (excluding an > unspecified set of system processes) for which the process has > permission to send that signal. and > The kill() function is successful if the process has permission to > send sig to any of the processes specified by pid. If kill() fails, > no signal shall be sent. and > [EPERM] > The process does not have permission to send the signal to any > receiving process. My reading of this is that kill(-1, ...) may fail with EPERM when there are no other processes to kill (since the current process is ignored). Since kill(-1, ...) only attempts to kill processes the user has permission to kill, it can't mean that we tried to do something we didn't have permission to kill, so it should be fine to interpret EPERM the same as success here for any POSIX-compliant system. This fixes an issue that Mic92 encountered[1] when he tried to review a Nixpkgs PR on FreeBSD. [1]: https://github.com/NixOS/nixpkgs/pull/81459#issuecomment-606073668 --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 89f7b58f8..ef37275ac 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -946,7 +946,7 @@ void killUser(uid_t uid) #else if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ if (errno != EINTR) throw SysError("cannot kill processes for uid '%1%'", uid); } From 37352aa7e19e0bfebbd0c32985cbf79a83508538 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 7 Feb 2021 20:44:56 +0100 Subject: [PATCH 105/725] Support --no-net for backwards compatibility --- src/libutil/args.cc | 3 +++ src/libutil/args.hh | 1 + src/nix/main.cc | 1 + 3 files changed, 5 insertions(+) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 71bae0504..9377fe4c0 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -14,6 +14,8 @@ void Args::addFlag(Flag && flag_) assert(flag->handler.arity == flag->labels.size()); assert(flag->longName != ""); longFlags[flag->longName] = flag; + for (auto & alias : flag->aliases) + longFlags[alias] = flag; if (flag->shortName) shortFlags[flag->shortName] = flag; } @@ -191,6 +193,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 42d8515ef..88f068087 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -97,6 +97,7 @@ protected: typedef std::shared_ptr ptr; std::string longName; + std::set aliases; char shortName = 0; std::string description; std::string category; diff --git a/src/nix/main.cc b/src/nix/main.cc index e95b04d85..ef5e41a55 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "offline", + .aliases = {"no-net"}, // FIXME: remove .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); From bab3f30755490207446966e9e828119462b57141 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Mon, 8 Feb 2021 11:49:07 +0100 Subject: [PATCH 106/725] Auto closing issues/PRs after 1year. --- .github/stale.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index f81b4c762..fe24942f4 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,9 +1,10 @@ # Configuration for probot-stale - https://github.com/probot/stale daysUntilStale: 180 -daysUntilClose: false +daysUntilClose: 365 exemptLabels: - "critical" -staleLabel: "2.status: stale" +staleLabel: "stale" markComment: | I marked this as stale due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) -closeComment: false +closeComment: | + I closed this issue due to inactivity. → [More info](https://github.com/NixOS/nix/blob/master/.github/STALE-BOT.md) From f2245091d033a8037aeb29ae701d20611500af6d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Feb 2021 12:26:41 -0500 Subject: [PATCH 107/725] Revert "narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash" --- src/libstore/binary-cache-store.cc | 6 +++++- tests/binary-cache.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 15163ead5..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,7 +176,11 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index f8d47170f..6697ce236 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -60,7 +60,7 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar | head -n1) +nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar From ad337c8697099ac9deb6e0ac16ea91d8acc51e4f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Feb 2021 17:33:28 +0000 Subject: [PATCH 108/725] Deeper `Command` hierarchy to remove redundancy Simply put, we now have `StorePathCommand : public StorePathsCommand` so `StorePathCommand` doesn't reimplement work. --- src/libcmd/command.cc | 4 +--- src/libcmd/command.hh | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index efdc98d5a..d29954f67 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -118,10 +118,8 @@ void StorePathsCommand::run(ref store, std::vector paths) run(store, std::move(storePaths)); } -void StorePathCommand::run(ref store) +void StorePathCommand::run(ref store, std::vector storePaths) { - auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables); - if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8c0b3a94a..c02193924 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -177,13 +177,13 @@ struct StorePathsCommand : public RealisedPathsCommand }; /* A command that operates on exactly one store path. */ -struct StorePathCommand : public InstallablesCommand +struct StorePathCommand : public StorePathsCommand { - using StoreCommand::run; + using StorePathsCommand::run; virtual void run(ref store, const StorePath & storePath) = 0; - void run(ref store) override; + void run(ref store, std::vector storePaths) override; }; /* A helper class for registering commands globally. */ From 35129884f9348f068d538e67bb559cc6104f714e Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Tue, 16 Feb 2021 23:19:42 +0000 Subject: [PATCH 109/725] Fix Haskell example http://nixos.org redirects to https://nixos.org and apparently the HTTP library doesn't follow the redirect, so the output is empty. When defining https in the request it crashes because the library doesn't seem to support https. So this switches the example to a different http library. --- doc/manual/src/command-ref/nix-shell.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 88b675e71..938d56e6e 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -232,22 +232,23 @@ terraform apply > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following -Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable +Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz +#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz -import Network.HTTP +import Network.Curl.Download import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) -- Fetch nixos.org and print all hrefs. main = do - resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/") - body <- getResponseBody resp - let tags = filter (isTagOpenName "a") $ parseTags body + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp let tags' = map (fromAttrib "href") tags mapM_ putStrLn $ filter (/= "") tags' ``` From 5f4701e70d35bb9ea2fb659caf387a30001e28ce Mon Sep 17 00:00:00 2001 From: Mauricio Scheffer Date: Tue, 16 Feb 2021 23:27:04 +0000 Subject: [PATCH 110/725] Update doc/manual/src/command-ref/nix-shell.md Co-authored-by: Cole Helbling --- doc/manual/src/command-ref/nix-shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 938d56e6e..54812a49f 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -238,7 +238,7 @@ branch): ```haskell #! /usr/bin/env nix-shell #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download import Text.HTML.TagSoup From 6042febfce3011aaa5e3c369ea14a0d93ad2880e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 15:30:49 +0100 Subject: [PATCH 111/725] Restore warning about 'nix' being experimental Fixes #4552. --- doc/manual/generate-manpage.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index a563c31f8..964b57086 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -7,7 +7,10 @@ let showCommand = { command, def, filename }: - "# Name\n\n" + '' + **Warning**: This program is **experimental** and its interface is subject to change. + '' + + "# Name\n\n" + "`${command}` - ${def.description}\n\n" + "# Synopsis\n\n" + showSynopsis { inherit command; args = def.args; } From 063de66909ff1b20394cdebdca1ef62bb6ca1d51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 16:42:03 +0100 Subject: [PATCH 112/725] nix develop: Fix quoted string handling Fixes #4540. --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3c44fdb0e..0938cbe5b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -59,7 +59,7 @@ BuildEnvironment readEnvironment(const Path & path) R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; static std::string squotedStringRegex = - R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; static std::string indexedArrayRegex = R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; From cced73496b835b545be91cbebc4f89f61a7b106f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 16:53:19 +0100 Subject: [PATCH 113/725] nix flake show: Handle 'overlays' output Fixes #4542. --- src/nix/flake.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4cd7d77a0..091af8084 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -880,7 +880,8 @@ struct CmdFlakeShow : FlakeCommand || attrPath[0] == "nixosConfigurations" || attrPath[0] == "nixosModules" || attrPath[0] == "defaultApp" - || attrPath[0] == "templates")) + || attrPath[0] == "templates" + || attrPath[0] == "overlays")) || ((attrPath.size() == 1 || attrPath.size() == 2) && (attrPath[0] == "checks" || attrPath[0] == "packages" @@ -943,7 +944,8 @@ struct CmdFlakeShow : FlakeCommand else { logger->cout("%s: %s", headerPrefix, - attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" : + (attrPath.size() == 1 && attrPath[0] == "overlay") + || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" : attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" : ANSI_YELLOW "unknown" ANSI_NORMAL); From f33878b6562c746d5865a86e64f02c75feaf5b3e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:11:14 +0100 Subject: [PATCH 114/725] Make 'nix --version -vv' work Fixes #3743. --- src/nix/main.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index ef5e41a55..5f4eb8918 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -61,6 +61,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs bool printBuildLogs = false; bool useNet = true; bool refresh = false; + bool showVersion = false; NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { @@ -87,7 +88,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "version", .description = "Show version information.", - .handler = {[&]() { if (!completions) printVersion(programName); }}, + .handler = {[&]() { showVersion = true; }}, }); addFlag({ @@ -280,6 +281,11 @@ void mainWrapped(int argc, char * * argv) initPlugins(); + if (args.showVersion) { + printVersion(programName); + return; + } + if (!args.command) throw UsageError("no subcommand specified"); From 13897afbe6cf7ef8013c0c94109696bb7b13d0c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:32:10 +0100 Subject: [PATCH 115/725] Throw an error if --arg / --argstr is used with a flake Fixes #3949. --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++-- src/libcmd/installables.hh | 12 +++++++----- src/nix/bundle.cc | 2 +- src/nix/develop.cc | 1 + src/nix/flake.cc | 2 +- src/nix/profile.cc | 8 +++++++- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9ad02b5f0..4739dc974 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -496,6 +496,23 @@ static std::string showAttrPaths(const std::vector & paths) return s; } +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + Strings && attrPaths, + Strings && prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(attrPaths), + prefixes(prefixes), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + std::tuple InstallableFlake::toDerivation() { auto lockedFlake = getLockedFlake(); @@ -628,9 +645,12 @@ std::vector> SourceExprCommand::parseInstallables( try { auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); result.push_back(std::make_shared( - getEvalState(), std::move(flakeRef), + this, + getEvalState(), + std::move(flakeRef), fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment}, - getDefaultFlakeAttrPathPrefixes(), lockFlags)); + getDefaultFlakeAttrPathPrefixes(), + lockFlags)); continue; } catch (...) { ex = std::current_exception(); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index f37b3f829..b714f097b 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -104,11 +104,13 @@ struct InstallableFlake : InstallableValue const flake::LockFlags & lockFlags; mutable std::shared_ptr _lockedFlake; - InstallableFlake(ref state, FlakeRef && flakeRef, - Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags) - : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths), - prefixes(prefixes), lockFlags(lockFlags) - { } + InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + Strings && attrPaths, + Strings && prefixes, + const flake::LockFlags & lockFlags); std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 1789e4598..48f4eb6e3 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -74,7 +74,7 @@ struct CmdBundle : InstallableCommand auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; - auto bundler = InstallableFlake( + auto bundler = InstallableFlake(this, evalState, std::move(bundlerFlakeRef), Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, Strings({"bundlers."}), lockFlags); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 0938cbe5b..d0b140570 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,6 +443,7 @@ struct CmdDevelop : Common, MixEnvironment auto state = getEvalState(); auto bashInstallable = std::make_shared( + this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 091af8084..b9cde5d6d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -595,7 +595,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); - auto installable = InstallableFlake( + auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), Strings{templateName == "" ? "defaultTemplate" : templateName}, Strings(attrsPathPrefixes), lockFlags); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 827f8be5a..4d275f577 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -399,7 +399,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); - InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags); + InstallableFlake installable( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + {element.source->attrPath}, + {}, + lockFlags); auto [attrPath, resolvedRef, drv] = installable.toDerivation(); From 7bd9898d5ca72ed136032590745c56826317a328 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:54:13 +0100 Subject: [PATCH 116/725] nix run: Allow program name to be set in meta.mainProgram This is useful when the program name doesn't match the package name (e.g. ripgrep vs rg). Fixes #4498. --- src/nix/app.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/nix/app.cc b/src/nix/app.cc index 80acbf658..cf147c631 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -12,11 +12,16 @@ App Installable::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); + auto checkProgram = [&](const Path & program) + { + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); + }; + if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - if (!state.store->isInStore(program)) - throw Error("app program '%s' is not in the Nix store", program); + checkProgram(program); std::vector context2; for (auto & [path, name] : context) @@ -33,9 +38,17 @@ App Installable::toApp(EvalState & state) auto outPath = cursor->getAttr(state.sOutPath)->getString(); auto outputName = cursor->getAttr(state.sOutputName)->getString(); auto name = cursor->getAttr(state.sName)->getString(); + auto aMeta = cursor->maybeGetAttr("meta"); + auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; + auto mainProgram = + aMainProgram + ? aMainProgram->getString() + : DrvName(name).name; + auto program = outPath + "/bin/" + mainProgram; + checkProgram(program); return App { .context = { { drvPath, {outputName} } }, - .program = outPath + "/bin/" + DrvName(name).name, + .program = program, }; } From 1b578255245e2e1347059ad7d9171cf822c723a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Feb 2021 17:58:40 +0100 Subject: [PATCH 117/725] Document meta.mainProgram Issue #4498. --- src/nix/run.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nix/run.md b/src/nix/run.md index c178e8b13..a76750376 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -43,9 +43,10 @@ program specified by the app definition. If *installable* evaluates to a derivation, it will try to execute the program `/bin/`, where *out* is the primary output store -path of the derivation and *name* is the name part of the value of the -`name` attribute of the derivation (e.g. if `name` is set to -`hello-1.10`, it will run `$out/bin/hello`). +path of the derivation and *name* is the `meta.mainProgram` attribute +of the derivation if it exists, and otherwise the name part of the +value of the `name` attribute of the derivation (e.g. if `name` is set +to `hello-1.10`, it will run `$out/bin/hello`). # Flake output attributes From cd44c0af71ace2eb8056c2b26b9249a5aa102b41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2021 19:22:37 +0100 Subject: [PATCH 118/725] Increase default stack size on Linux Workaround for #4550. --- src/nix/main.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5f4eb8918..1b68cf15b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -17,6 +17,10 @@ #include #include +#if __linux__ +#include +#endif + #include extern std::string chrootHelperName; @@ -325,6 +329,17 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + #if __linux__ + rlim_t stackSize = 64 * 1024 * 1024; + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif + return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); From 263f6dbd1cef6eb9560737f6daf963f8968a65d8 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 20:38:37 +0100 Subject: [PATCH 119/725] Don't crash nix-build when not all outputs are realised Change the `nix-build` logic for linking/printing the output paths to allow for some outputs to be missing. This might happen when the toplevel derivation didn't have to be built, either because all the required outputs were already there, or because they have all been substituted. --- src/nix-build/nix-build.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 361f9730d..d975cd16d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -518,9 +518,11 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = store->queryDerivationOutputMap(drvPath); + auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath); - auto outputPath = builtOutputs.at(outputName); + auto maybeOutputPath = builtOutputs.at(outputName); + assert(maybeOutputPath); + auto outputPath = *maybeOutputPath; if (auto store2 = store.dynamic_pointer_cast()) { std::string symlink = drvPrefix; From be1b5c4e59ca1c3504a44e2058807f7207432846 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 18:11:33 +0100 Subject: [PATCH 120/725] Test the garbage collection of CA derivations Simple test to ensure that `nix-build && nix-collect-garbage && nix-build -j0` works as it should --- tests/content-addressed.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index e8ac88609..7e32e1f28 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -48,6 +48,10 @@ testCutoff () { testGC () { nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true + clearStore + buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA + nix-collect-garbage --experimental-features ca-derivations + buildAttr rootCA 1 -j0 } testNixCommand () { From 87c8d3d702123528ac068bb703232e24431c535e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 27 Jan 2021 10:03:05 +0100 Subject: [PATCH 121/725] Register the realisations for unresolved drvs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once a build is done, get back to the original derivation, and register all the newly built outputs for this derivation. This allows Nix to work properly with derivations that don't have all their build inputs available − thus allowing garbage collection and (once it's implemented) binary substitution --- src/libstore/build/derivation-goal.cc | 54 ++++++++++++++++++++++++++- src/libstore/build/derivation-goal.hh | 3 ++ src/libstore/derivations.cc | 9 ++++- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 15 +------- src/libstore/store-api.hh | 6 --- 7 files changed, 67 insertions(+), 24 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index eeaec4f2c..315cf3f0a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -506,6 +506,7 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); + resolvedDrv = drvResolved; auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -1019,7 +1020,45 @@ void DerivationGoal::buildDone() } void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); + assert(resolvedDrv); + + // If the derivation was originally a full `Derivation` (and not just + // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` + // will be wrong otherwise + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + + auto originalHashes = staticOutputHashes(worker.store, fullDrv); + auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv->outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(originalHashes.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = worker.store.queryRealisation( + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + ); + // We've just built it, but maybe the build failed, in which case the + // realisation won't be there + if (realisation) { + auto newRealisation = *realisation; + newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + worker.store.registerDrvOutput(newRealisation); + } else { + // If we don't have a realisation, then it must mean that something + // failed when building the resolved drv + assert(!result.success()); + } + } + + // This is potentially a bit fishy in terms of error reporting. Not sure + // how to do it in a cleaner way + amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); } HookReply DerivationGoal::tryBuildHook() @@ -3804,6 +3843,19 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + auto outputHashes = staticOutputHashes(worker.store, fullDrv); + if (auto real = worker.store.queryRealisation( + DrvOutput{outputHashes.at(i.first), i.first})) { + info.known = { + .path = real->outPath, + .status = PathStatus::Valid, + }; + } + } initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 8ee0be9e1..b7b85c85d 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -48,6 +48,9 @@ struct DerivationGoal : public Goal /* The path of the derivation. */ StorePath drvPath; + /* The path of the corresponding resolved derivation */ + std::optional resolvedDrv; + /* The specific outputs that we need to build. Empty means all of them. */ StringSet wantedOutputs; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7466c7d41..4b774c42a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -756,8 +756,13 @@ std::optional Derivation::tryResolveUncached(Store & store) { StringSet newOutputNames; for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) + if (!actualPathOpt) { + warn("Input %s!%s missing, aborting the resolving", + store.printStorePath(input.first), + outputName + ); return std::nullopt; + } auto actualPath = *actualPathOpt; inputRewrites.emplace( downstreamPlaceholder(store, input.first, outputName), @@ -782,6 +787,8 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; + debug("Trying to resolve %s", store.printStorePath(drvPath)); + { auto resolutions = resolutionsCache.lock(); auto resolvedDrvIter = resolutions->find(drvPath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f45af2bac..e06c47cde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9d235ba0a..780cc0f07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 37c11fe86..2658f7617 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,7 +366,7 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); @@ -376,19 +376,6 @@ std::map> Store::queryDerivationOutputMapN return outputs; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) -{ - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto resolvedDrv = Derivation::tryResolve(*this, path); - if (resolvedDrv) { - auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); - if (isValidPath(resolvedDrvPath)) - return queryDerivationOutputMapNoResolve(resolvedDrvPath); - } - } - return queryDerivationOutputMapNoResolve(path); -} - OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9e98eb8f9..6dcd43ed1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -415,12 +415,6 @@ public: `std::nullopt`. */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* - * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve - * the derivation - */ - virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ OutputPathMap queryDerivationOutputMap(const StorePath & path); From 93d9eb78a0733c5adcbc6ee7b8a257605ae4a32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:12:24 +0100 Subject: [PATCH 122/725] Syntactic fixes Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 4b774c42a..36993ffc2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -787,7 +787,7 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; - debug("Trying to resolve %s", store.printStorePath(drvPath)); + debug("trying to resolve %s", store.printStorePath(drvPath)); { auto resolutions = resolutionsCache.lock(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e06c47cde..0962418dd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { From 0bfbd043699908bcaff1493c733ab4798b642b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:13:38 +0100 Subject: [PATCH 123/725] Don't expose the "bang" drvoutput syntax It's not fixed nor useful atm, so better keep it hidden Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 36993ffc2..7807089ca 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -757,9 +757,9 @@ std::optional Derivation::tryResolveUncached(Store & store) { for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); if (!actualPathOpt) { - warn("Input %s!%s missing, aborting the resolving", - store.printStorePath(input.first), - outputName + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(input.first) ); return std::nullopt; } From 4bc28c44f258f4f8c8a3935d1acf746f6abe3d8f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 14:41:49 +0100 Subject: [PATCH 124/725] Store the output hashes in the initialOutputs of the drv goal That way we 1. Don't have to recompute them several times 2. Can compute them in a place where we know the type of the parent derivation, meaning that we don't need the casting dance we had before --- src/libstore/build/derivation-goal.cc | 49 ++++++++++++++++----------- src/libstore/build/derivation-goal.hh | 1 + 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 315cf3f0a..d8a89a2d0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -124,6 +124,17 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation , buildMode(buildMode) { this->drv = std::make_unique(BasicDerivation(drv)); + + auto outputHashes = staticOutputHashes(worker.store, drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", @@ -258,8 +269,20 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); + auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); + + auto outputHashes = staticOutputHashes(worker.store, *fullDrv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); + drv = std::unique_ptr(fullDrv); haveDerivation(); } @@ -1022,14 +1045,6 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { assert(resolvedDrv); - // If the derivation was originally a full `Derivation` (and not just - // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` - // will be wrong otherwise - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - - auto originalHashes = staticOutputHashes(worker.store, fullDrv); auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); // `wantedOutputs` might be empty, which means “all the outputs” @@ -1038,7 +1053,7 @@ void DerivationGoal::resolvedFinished() { realWantedOutputs = resolvedDrv->outputNames(); for (auto & wantedOutput : realWantedOutputs) { - assert(originalHashes.count(wantedOutput) != 0); + assert(initialOutputs.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0); auto realisation = worker.store.queryRealisation( DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} @@ -1047,7 +1062,7 @@ void DerivationGoal::resolvedFinished() { // realisation won't be there if (realisation) { auto newRealisation = *realisation; - newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something @@ -3829,9 +3844,8 @@ void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; + InitialOutput & info = initialOutputs.at(i.first); + info.wanted = wantOutput(i.first, wantedOutputs); if (i.second) { auto outputPath = *i.second; info.known = { @@ -3844,19 +3858,14 @@ void DerivationGoal::checkPathValidity() }; } if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - auto outputHashes = staticOutputHashes(worker.store, fullDrv); if (auto real = worker.store.queryRealisation( - DrvOutput{outputHashes.at(i.first), i.first})) { + DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; } } - initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index b7b85c85d..761100d3a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -37,6 +37,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; + Hash outputHash; std::optional known; }; From f483b623e98a0feb2568e5be076b533c5838ba32 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Feb 2021 08:16:12 +0100 Subject: [PATCH 125/725] Remove the drv resolution caching mechanism It isn't needed anymore now that don't need to eagerly resolve everything like we used to do. So we can safely get rid of it --- src/libstore/derivations.cc | 34 +--------------------------------- src/libstore/derivations.hh | 4 ---- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7807089ca..6d0742b4f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolveUncached(Store & store) { +std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -776,36 +776,4 @@ std::optional Derivation::tryResolveUncached(Store & store) { return resolved; } -std::optional Derivation::tryResolve(Store& store) -{ - auto drvPath = writeDerivation(store, *this, NoRepair, false); - return Derivation::tryResolve(store, drvPath); -} - -std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - - debug("trying to resolve %s", store.printStorePath(drvPath)); - - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(drvPath); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(drvPath); - auto attempt = drv.tryResolveUncached(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); - return *attempt; -} - } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 3d8f19aef..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,14 +138,10 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); - static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - -private: - std::optional tryResolveUncached(Store & store); }; From ae4260f0a79c5cbb7c88ddbef1f512e0771f7414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 15 Feb 2021 10:20:54 +0000 Subject: [PATCH 126/725] Generate installer script for each PR/push This works by using Cachix feature of serving a file from a store path. --- .github/workflows/test.yml | 44 +++++++++++- flake.nix | 70 ++++++++++---------- scripts/prepare-installer-for-github-actions | 10 +++ 3 files changed, 89 insertions(+), 35 deletions(-) create mode 100755 scripts/prepare-installer-for-github-actions diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 021642f4c..bde6106e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,10 +8,52 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} + env: + CACHIX_NAME: nix-ci steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v12 + - uses: cachix/cachix-action@v8 + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) + installer: + if: github.event_name == 'push' + needs: tests + runs-on: ubuntu-latest + env: + CACHIX_NAME: nix-ci + outputs: + installerURL: ${{ steps.prepare-installer.outputs.installerURL }} + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v12 + - uses: cachix/cachix-action@v8 + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + - id: prepare-installer + run: scripts/prepare-installer-for-github-actions + installer_test: + if: github.event_name == 'push' + needs: installer + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + CACHIX_NAME: nix-ci + steps: + - uses: actions/checkout@v2.3.4 + - uses: cachix/install-nix-action@master + with: + install_url: '${{needs.installer.outputs.installerURL}}' + install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve' + - run: nix-instantiate -E 'builtins.currentTime' --eval + \ No newline at end of file diff --git a/flake.nix b/flake.nix index 8c60934e6..fc334ac5b 100644 --- a/flake.nix +++ b/flake.nix @@ -109,6 +109,40 @@ ]; }; + installScriptFor = systems: + with nixpkgsFor.x86_64-linux; + runCommand "installer-script" + { buildInputs = [ nix ]; + } + '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix + # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ + '' + ) + systems + } --replace '@nixVersion@' ${version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -313,40 +347,8 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = - with nixpkgsFor.x86_64-linux; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix - # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (system: - '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \ - '' - ) - [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] - } \ - --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. coverage = diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions new file mode 100755 index 000000000..92d930384 --- /dev/null +++ b/scripts/prepare-installer-for-github-actions @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) +installerHash=$(echo $script | cut -b12-43 -) + +installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install + +echo "::set-output name=installerURL::$installerURL" From 22aec8cef43e77bba356d099868fe0a6e7545b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sun, 21 Feb 2021 14:55:45 +0000 Subject: [PATCH 127/725] fix installer script --- scripts/install.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 0eaf25bb3..7d25f7bd7 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -60,7 +60,7 @@ case "$(uname -s).$(uname -m)" in esac # Use this command-line option to fetch the tarballs using nar-serve or Cachix -if "${1:---tarball-url-prefix}"; then +if [ "${1:-}" = "--tarball-url-prefix" ]; then if [ -z "${2:-}" ]; then oops "missing argument for --tarball-url-prefix" fi From 2de232d2b301b2f0854b9fa715ab085612c85e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 16 Feb 2021 14:32:12 +0100 Subject: [PATCH 128/725] Add x86_64 compute levels as additional system types When performing distributed builds of machine learning packages, it would be nice if builders without the required SIMD instructions can be excluded as build nodes. Since x86_64 has accumulated a large number of different instruction set extensions, listing all possible extensions would be unwieldy. AMD, Intel, Red Hat, and SUSE have recently defined four different microarchitecture levels that are now part of the x86-64 psABI supplement and will be used in glibc 2.33: https://gitlab.com/x86-psABIs/x86-64-ABI https://lwn.net/Articles/844831/ This change uses libcpuid to detect CPU features and then uses them to add the supported x86_64 levels to the additional system types. For example on a Ryzen 3700X: $ ~/aps/bin/nix -vv --version | grep "Additional system" Additional system types: i686-linux, x86_64-v1-linux, x86_64-v2-linux, x86_64-v3-linux --- Makefile.config.in | 1 + configure.ac | 8 ++++ flake.nix | 3 +- src/libstore/globals.cc | 24 +++++++---- src/libutil/compute-levels.cc | 80 +++++++++++++++++++++++++++++++++++ src/libutil/compute-levels.hh | 7 +++ src/libutil/local.mk | 4 ++ tests/compute-levels.sh | 7 +++ tests/local.mk | 3 +- 9 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 src/libutil/compute-levels.cc create mode 100644 src/libutil/compute-levels.hh create mode 100644 tests/compute-levels.sh diff --git a/Makefile.config.in b/Makefile.config.in index d1e59e4e7..9d0500e48 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,6 +9,7 @@ CXXFLAGS = @CXXFLAGS@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ GTEST_LIBS = @GTEST_LIBS@ +HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ diff --git a/configure.ac b/configure.ac index 2047ed8d2..a24287ff6 100644 --- a/configure.ac +++ b/configure.ac @@ -218,6 +218,14 @@ LDFLAGS="-lz $LDFLAGS" # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) +# Look for libcpuid. +if test "$machine_name" = "x86_64"; then + PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"]) + have_libcpuid=1 + AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid]) +fi +AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) + # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then diff --git a/flake.nix b/flake.nix index 8c60934e6..3ad7cca97 100644 --- a/flake.nix +++ b/flake.nix @@ -91,7 +91,8 @@ gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isx86_64 libcpuid; awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0531aad9f..df07aee9b 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -3,6 +3,7 @@ #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "compute-levels.hh" #include #include @@ -133,24 +134,29 @@ StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultExtraPlatforms() { + StringSet extraPlatforms; + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) - return StringSet{"i686-linux"}; -#if __APPLE__ + extraPlatforms.insert("i686-linux"); + +#if __linux__ + StringSet levels = computeLevels(); + for (auto iter = levels.begin(); iter != levels.end(); ++iter) + extraPlatforms.insert(*iter + "-linux"); +#elif __APPLE__ // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") - return StringSet{"aarch64-darwin"}; + extraPlatforms.insert("aarch64-darwin"); else if (std::string{SYSTEM} == "aarch64-darwin") - return StringSet{"x86_64-darwin"}; - else - return StringSet{}; + extraPlatforms.insert("x86_64-darwin"); } #endif - else - return StringSet{}; + + return extraPlatforms; } bool Settings::isExperimentalFeatureEnabled(const std::string & name) diff --git a/src/libutil/compute-levels.cc b/src/libutil/compute-levels.cc new file mode 100644 index 000000000..19eaedfa8 --- /dev/null +++ b/src/libutil/compute-levels.cc @@ -0,0 +1,80 @@ +#include "types.hh" + +#if HAVE_LIBCPUID +#include +#endif + +namespace nix { + +#if HAVE_LIBCPUID + +StringSet computeLevels() { + StringSet levels; + + if (!cpuid_present()) + return levels; + + cpu_raw_data_t raw; + cpu_id_t data; + + if (cpuid_get_raw_data(&raw) < 0) + return levels; + + if (cpu_identify(&raw, &data) < 0) + return levels; + + if (!(data.flags[CPU_FEATURE_CMOV] && + data.flags[CPU_FEATURE_CX8] && + data.flags[CPU_FEATURE_FPU] && + data.flags[CPU_FEATURE_FXSR] && + data.flags[CPU_FEATURE_MMX] && + data.flags[CPU_FEATURE_SSE] && + data.flags[CPU_FEATURE_SSE2])) + return levels; + + levels.insert("x86_64-v1"); + + if (!(data.flags[CPU_FEATURE_CX16] && + data.flags[CPU_FEATURE_LAHF_LM] && + data.flags[CPU_FEATURE_POPCNT] && + // SSE3 + data.flags[CPU_FEATURE_PNI] && + data.flags[CPU_FEATURE_SSSE3] && + data.flags[CPU_FEATURE_SSE4_1] && + data.flags[CPU_FEATURE_SSE4_2])) + return levels; + + levels.insert("x86_64-v2"); + + if (!(data.flags[CPU_FEATURE_AVX] && + data.flags[CPU_FEATURE_AVX2] && + data.flags[CPU_FEATURE_F16C] && + data.flags[CPU_FEATURE_FMA3] && + // LZCNT + data.flags[CPU_FEATURE_ABM] && + data.flags[CPU_FEATURE_MOVBE])) + return levels; + + levels.insert("x86_64-v3"); + + if (!(data.flags[CPU_FEATURE_AVX512F] && + data.flags[CPU_FEATURE_AVX512BW] && + data.flags[CPU_FEATURE_AVX512CD] && + data.flags[CPU_FEATURE_AVX512DQ] && + data.flags[CPU_FEATURE_AVX512VL])) + return levels; + + levels.insert("x86_64-v4"); + + return levels; +} + +#else + +StringSet computeLevels() { + return StringSet{}; +} + +#endif // HAVE_LIBCPUID + +} diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh new file mode 100644 index 000000000..8ded295f9 --- /dev/null +++ b/src/libutil/compute-levels.hh @@ -0,0 +1,7 @@ +#include "types.hh" + +namespace nix { + +StringSet computeLevels(); + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index ae7eb67ad..5341c58e6 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context + +ifeq ($(HAVE_LIBCPUID), 1) + libutil_LDFLAGS += -lcpuid +endif diff --git a/tests/compute-levels.sh b/tests/compute-levels.sh new file mode 100644 index 000000000..e4322dfa1 --- /dev/null +++ b/tests/compute-levels.sh @@ -0,0 +1,7 @@ +source common.sh + +if [[ $(uname -ms) = "Linux x86_64" ]]; then + # x86_64 CPUs must always support the baseline + # microarchitecture level. + nix -vv --version | grep -q "x86_64-v1-linux" +fi diff --git a/tests/local.mk b/tests/local.mk index aa8b4f9bf..06be8cec1 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,7 +38,8 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ content-addressed.sh \ - build.sh + build.sh \ + compute-levels.sh # parallel.sh # build-remote-content-addressed-fixed.sh \ From 574eb2be81cc599162722659dcb95f19173c98d1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2021 15:24:14 +0100 Subject: [PATCH 129/725] Tweak error message --- src/libexpr/eval.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7271776eb..e2f2308aa 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1381,10 +1381,10 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') -nix attempted to evaluate a function as a top level expression; in this case it must have its -arguments supplied either by default values, or passed explicitly with --arg or --argstr. - -https://nixos.org/manual/nix/stable/#ss-functions)", i.name); +Nix attempted to evaluate a function as a top level expression; in +this case it must have its arguments supplied either by default +values, or passed explicitly with '--arg' or '--argstr'. See +https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); } } From e2f3b2eb42a0ceca36ce00973bd2d49b1a3e6a2c Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 22 Feb 2021 16:13:09 +0100 Subject: [PATCH 130/725] Make missing auto-call arguments throw an eval error The PR #4240 changed messag of the error that was thrown when an auto-called function was missing an argument. However this change also changed the type of the error, from `EvalError` to a new `MissingArgumentError`. This broke hydra which was relying on an `EvalError` being thrown. Make `MissingArgumentError` a subclass of `EvalError` to un-break hydra. --- src/libexpr/nixexpr.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index cbe9a45bf..8df8055b3 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,7 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); -MakeError(MissingArgumentError, Error); +MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); From 35205e2e922952fc0654260a07fc3191c5afc2cc Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 22 Feb 2021 17:10:55 -0500 Subject: [PATCH 131/725] Warn about instability of plugin API --- src/libstore/globals.hh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1d968ef3e..1254698ca 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -831,6 +831,9 @@ public: command, and RegisterSetting to add new nix config settings. See the constructors for those types for more details. + Warning! These APIs are inherently unstable and may change from + release to release. + Since these files are loaded into the same address space as Nix itself, they must be DSOs compatible with the instance of Nix running at the time (i.e. compiled against the same headers, not From 6fbf3fe636bc1d9a9aba4bacb2a70191c1d6b1a7 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:48:41 +0100 Subject: [PATCH 132/725] Make the build-hook work with ca derivations - Pass it the name of the outputs rather than their output paths (as these don't exist for ca derivations) - Get the built output paths from the remote builder - Register the new received realisations --- src/build-remote/build-remote.cc | 36 +++++++++++++++++++++------ src/libstore/build/derivation-goal.cc | 9 ++++--- src/libstore/realisation.hh | 2 ++ src/libstore/remote-store.cc | 16 ++++++++++++ src/libstore/store-api.hh | 2 ++ src/libstore/worker-protocol.hh | 2 ++ 6 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5b8ab3387..c2319a3d1 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -248,7 +248,7 @@ connected: std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); - auto outputs = readStrings(source); + auto wantedOutputs = readStrings(source); AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); @@ -273,6 +273,7 @@ connected: uploadLock = -1; auto drv = store->readDerivation(*drvPath); + auto outputHashes = staticOutputHashes(*store, drv); drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); @@ -280,16 +281,35 @@ connected: if (!result.success()) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); - StorePathSet missing; - for (auto & path : outputs) - if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); + std::set missingRealisations; + StorePathSet missingPaths; + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & outputName : wantedOutputs) { + auto thisOutputHash = outputHashes.at(outputName); + auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; + if (!store->queryRealisation(thisOutputId)) { + notice("Missing output %s", outputName); + assert(result.builtOutputs.count(thisOutputId)); + auto newRealisation = result.builtOutputs.at(thisOutputId); + missingRealisations.insert(newRealisation); + missingPaths.insert(newRealisation.outPath); + } + } + } else { + auto outputPaths = drv.outputsAndOptPaths(*store); + for (auto & [outputName, hopefullyOutputPath] : outputPaths) { + assert(hopefullyOutputPath.second); + if (!store->isValidPath(*hopefullyOutputPath.second)) + missingPaths.insert(*hopefullyOutputPath.second); + } + } - if (!missing.empty()) { + if (!missingPaths.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) - for (auto & i : missing) - localStore->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ - copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); + for (auto & path : missingPaths) + localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ + copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } return 0; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8a89a2d0..b074410b0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1159,13 +1159,14 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ { - StorePathSet missingPaths; - for (auto & [_, status] : initialOutputs) { + StringSet missingOutputs; + for (auto & [outputName, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - missingPaths.insert(status.known->path); + missingOutputs.insert(outputName); + /* missingPaths.insert(status.known->path); */ } - worker_proto::write(worker.store, hook->sink, missingPaths); + worker_proto::write(worker.store, hook->sink, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 7c91d802a..fc92d3c17 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -33,6 +33,8 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +typedef std::map DrvOutputs; + struct OpaquePath { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be07f02dc..52d633372 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -12,6 +12,7 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include namespace nix { @@ -49,6 +50,21 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } +Realisation read(const Store & store, Source & from, Phantom _) +{ + std::string rawInput = readString(from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} +void write(const Store & store, Sink & out, const Realisation & realisation) +{ out << realisation.toJSON().dump(); } + +DrvOutput read(const Store & store, Source & from, Phantom _) +{ return DrvOutput::parse(readString(from)); } +void write(const Store & store, Sink & out, const DrvOutput & drvOutput) +{ out << drvOutput.to_string(); } std::optional read(const Store & store, Source & from, Phantom> _) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6dcd43ed1..ea6389ba4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -162,6 +162,8 @@ struct BuildResult non-determinism.) */ bool isNonDeterministic = false; + DrvOutputs builtOutputs; + /* The start/stop times of the build (or one of the rounds, if it was repeated). */ time_t startTime = 0, stopTime = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f2cdc7ca3..5e094c378 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,6 +86,8 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, Realisation); +MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(template, std::set); From 5687564a27bee692f68a78b897a2d68715f6a3ce Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:50:44 +0100 Subject: [PATCH 133/725] LocalStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/build/entry-points.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 9f97d40ba..99b3fa070 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -58,6 +58,26 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat result.status = BuildResult::MiscFailure; result.errorMsg = e.msg(); } + // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's + // extended to return the full realisation for each output + auto staticDrvOutputs = drv.outputsAndOptPaths(*this); + auto outputHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, staticOutput] : staticDrvOutputs) { + auto outputId = DrvOutput{outputHashes.at(outputName), outputName}; + if (staticOutput.second) + result.builtOutputs.insert_or_assign( + outputId, + Realisation{ outputId, *staticOutput.second} + ); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto realisation = this->queryRealisation(outputId); + if (realisation) + result.builtOutputs.insert_or_assign( + outputId, + *realisation + ); + } + } return result; } From a2b69660a9b326b95d48bd222993c5225bbd5b5f Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:50:44 +0100 Subject: [PATCH 134/725] LegacySSHStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/legacy-ssh-store.cc | 4 +++- src/libstore/serve-protocol.hh | 2 +- src/nix-store/nix-store.cc | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 253c0033e..daf78042f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -258,7 +258,9 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { + status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + } return status; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 9fae6d534..0a17387cb 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x205 +#define SERVE_PROTOCOL_VERSION 0x206 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 37191b9e6..559fd5355 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -905,6 +905,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; + if (GET_PROTOCOL_MINOR(clientVersion >= 5)) { + worker_proto::write(*store, out, status.builtOutputs); + } + break; } From 27b5747ca7b5599768083dde5fa4d36bfbb0f66f Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 25 Jan 2021 11:08:38 +0100 Subject: [PATCH 135/725] RemoteStore: Send back the new realisations To allow it to build ca derivations remotely --- src/libstore/daemon.cc | 3 +++ src/libstore/remote-store.cc | 4 ++++ src/libstore/worker-protocol.hh | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ba5788b64..ba7959263 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -575,6 +575,9 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; + if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) { + worker_proto::write(*store, to, res.builtOutputs); + } break; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 52d633372..0d884389a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -680,6 +680,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD unsigned int status; conn->from >> status >> res.errorMsg; res.status = (BuildResult::Status) status; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) { + auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + res.builtOutputs = builtOutputs; + } return res; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5e094c378..95f08bc9a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11b +#define PROTOCOL_VERSION 0x11c #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) From 8c385d16eeeb26a912d213c5689d9f9a78020bc7 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 09:35:10 +0100 Subject: [PATCH 136/725] Also send ca outputs to the build hook Otherwise they don't get registered, triggering an assertion failure at some point later --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b074410b0..096f24029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,8 +1161,8 @@ HookReply DerivationGoal::tryBuildHook() { StringSet missingOutputs; for (auto & [outputName, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; + // XXX: Does this include known CA outputs? + if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); /* missingPaths.insert(status.known->path); */ } From 69666b951ee06733ed420cb4cd408a19e42c6e43 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 09:36:24 +0100 Subject: [PATCH 137/725] build-remote: Always register the missing outputs It's possible that all the paths are already there, but just not associated to the current drv output --- src/build-remote/build-remote.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c2319a3d1..228aba35a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,6 +311,13 @@ connected: localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } + // XXX: Should e done as part of `copyPaths` + for (auto & realisation : missingRealisations) { + // Should hold, because if the feature isn't enabled the set + // of missing realisations should be empty + settings.requireExperimentalFeature("ca-derivations"); + store->registerDrvOutput(realisation); + } return 0; } From 527da736905730e70725bf4b3556d61267d220ba Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:02:03 +0100 Subject: [PATCH 138/725] Properly bypass the registering step when all outputs are present There was already some logic for that, but it didn't handle the case of content-addressed outputs, so extend it a bit for that --- src/libstore/build/derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 096f24029..6052b625d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3001,11 +3001,11 @@ void DerivationGoal::registerOutputs() */ if (hook) { bool allValid = true; - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (!i.second.second || !worker.store.isValidPath(*i.second.second)) + for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!outputPath || !worker.store.isValidPath(*outputPath)) allValid = false; else - finalOutputs.insert_or_assign(i.first, *i.second.second); + finalOutputs.insert_or_assign(outputName, *outputPath); } if (allValid) return; } From c32168c9bc161e0c9cea027853895971699510cb Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 26 Jan 2021 10:28:00 +0100 Subject: [PATCH 139/725] Test the remote building of ca derivations --- tests/build-hook-ca.nix | 16 ++++++++++++---- tests/build-remote-content-addressed-fixed.sh | 5 ----- tests/build-remote-content-addressed-floating.sh | 7 +++++++ tests/local.mk | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) delete mode 100644 tests/build-remote-content-addressed-fixed.sh create mode 100644 tests/build-remote-content-addressed-floating.sh diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca.nix index 98db473fc..67295985f 100644 --- a/tests/build-hook-ca.nix +++ b/tests/build-hook-ca.nix @@ -11,6 +11,7 @@ let args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; + __contentAddressed = true; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; @@ -19,7 +20,6 @@ let name = "build-remote-input-1"; buildCommand = "echo FOO > $out"; requiredSystemFeatures = ["foo"]; - outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; }; input2 = mkDerivation { @@ -27,7 +27,16 @@ let name = "build-remote-input-2"; buildCommand = "echo BAR > $out"; requiredSystemFeatures = ["bar"]; - outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + read x < ${input2} + echo $x BAZ > $out + ''; + requiredSystemFeatures = ["baz"]; }; in @@ -38,8 +47,7 @@ in buildCommand = '' read x < ${input1} - read y < ${input2} + read y < ${input3} echo "$x $y" > $out ''; - outputHash = "sha256-3YGhlOfbGUm9hiPn2teXXTT8M1NEpDFvfXkxMaJRld0="; } diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh deleted file mode 100644 index 1408a19d5..000000000 --- a/tests/build-remote-content-addressed-fixed.sh +++ /dev/null @@ -1,5 +0,0 @@ -source common.sh - -file=build-hook-ca.nix - -source build-remote.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh new file mode 100644 index 000000000..cbb75729b --- /dev/null +++ b/tests/build-remote-content-addressed-floating.sh @@ -0,0 +1,7 @@ +source common.sh + +file=build-hook-ca.nix + +sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf + +source build-remote.sh diff --git a/tests/local.mk b/tests/local.mk index aa8b4f9bf..9bde2322f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + build-remote-content-addressed-floating.sh \ ssh-relay.sh \ nar-access.sh \ structured-attrs.sh \ @@ -40,7 +41,6 @@ nix_tests = \ content-addressed.sh \ build.sh # parallel.sh - # build-remote-content-addressed-fixed.sh \ install-tests += $(foreach x, $(nix_tests), tests/$(x)) From ba1a256d0875592b28d902f3e40663b2adedfe9c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 23 Feb 2021 14:12:11 +0100 Subject: [PATCH 140/725] Make `DerivationGoal::drv` a full Derivation This field used to be a `BasicDerivation`, but this `BasicDerivation` was downcasted to a `Derivation` when needed (implicitely or not), so we might as well make it a full `Derivation` and upcast it when needed. This also allows getting rid of a weird duplication in the way we compute the static output hashes for the derivation. We had to do it differently and in a different place depending on whether the derivation was a full derivation or just a basic drv, but we can now do it unconditionally on the full derivation. Fix #4559 --- src/libstore/build/derivation-goal.cc | 37 ++++++++++----------------- src/libstore/build/derivation-goal.hh | 2 +- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d8a89a2d0..804a79e4c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -123,17 +123,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - this->drv = std::make_unique(BasicDerivation(drv)); - - auto outputHashes = staticOutputHashes(worker.store, drv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); + this->drv = std::make_unique(drv); state = &DerivationGoal::haveDerivation; name = fmt( @@ -271,18 +261,8 @@ void DerivationGoal::loadDerivation() auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); - auto outputHashes = staticOutputHashes(worker.store, *fullDrv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); - /* Get the derivation. */ - drv = std::unique_ptr(fullDrv); + drv = std::unique_ptr(fullDrv); haveDerivation(); } @@ -301,6 +281,16 @@ void DerivationGoal::haveDerivation() if (i.second.second) worker.store.addTempRoot(*i.second.second); + auto outputHashes = staticOutputHashes(worker.store, *drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + /* Check what outputs paths are not already valid. */ checkPathValidity(); bool allValid = true; @@ -3517,10 +3507,9 @@ void DerivationGoal::registerOutputs() but it's fine to do in all cases. */ if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto outputHashes = staticOutputHashes(worker.store, *drv); for (auto& [outputName, newInfo] : infos) worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{outputHashes.at(outputName), outputName}, + .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, .outPath = newInfo.path}); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 761100d3a..6dc164922 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -64,7 +64,7 @@ struct DerivationGoal : public Goal bool retrySubstitution; /* The derivation stored at drvPath. */ - std::unique_ptr drv; + std::unique_ptr drv; std::unique_ptr parsedDrv; From ec3497c1d63f4c0547d0402d92015f846f56aac7 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 07:37:04 -0500 Subject: [PATCH 141/725] Bail if plugin-files is set after plugins have been loaded. We know the flag will be ignored but the user wants it to take effect. --- src/libstore/globals.cc | 11 +++++++++++ src/libstore/globals.hh | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index df07aee9b..03294b7fe 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -243,6 +243,14 @@ void MaxBuildJobsSetting::set(const std::string & str, bool append) } +void PluginFilesSetting::set(const std::string & str, bool append) +{ + if (pluginsLoaded) + throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + BaseSetting::set(str, append); +} + + void initPlugins() { for (const auto & pluginFile : settings.pluginFiles.get()) { @@ -270,6 +278,9 @@ void initPlugins() unknown settings. */ globalConfig.reapplyUnknownSettings(); globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + settings.pluginFiles.pluginsLoaded = true; } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1254698ca..df61d6417 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -28,6 +28,23 @@ struct MaxBuildJobsSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting(Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str, bool append = false) override; +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -819,7 +836,7 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting pluginFiles{ + PluginFilesSetting pluginFiles{ this, {}, "plugin-files", R"( A list of plugin files to be loaded by Nix. Each of these files will From 98d1b64400cc7b75216fc885859883c707c18bef Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 09:37:43 -0500 Subject: [PATCH 142/725] Initialize plugins after handling initial command line flags This is technically a breaking change, since attempting to set plugin files after the first non-flag argument will now throw an error. This is acceptable given the relative lack of stability in a plugin interface and the need to tie the knot somewhere once plugins can actually define new subcommands. --- doc/manual/src/release-notes/rl-2.4.md | 7 +++++++ src/build-remote/build-remote.cc | 3 +++ src/libmain/common-args.cc | 7 +++++++ src/libmain/common-args.hh | 6 +++++- src/libstore/globals.cc | 1 + src/libutil/args.cc | 8 ++++++++ src/libutil/args.hh | 4 ++++ src/nix-build/nix-build.cc | 2 -- src/nix-channel/nix-channel.cc | 2 -- src/nix-collect-garbage/nix-collect-garbage.cc | 2 -- src/nix-copy-closure/nix-copy-closure.cc | 2 -- src/nix-env/nix-env.cc | 2 -- src/nix-instantiate/nix-instantiate.cc | 2 -- src/nix-store/nix-store.cc | 2 -- src/nix/daemon.cc | 2 -- src/nix/main.cc | 2 -- src/nix/prefetch.cc | 2 -- tests/plugins.sh | 2 +- 18 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.4.md diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md new file mode 100644 index 000000000..26ba70904 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -0,0 +1,7 @@ +# Release 2.4 (202X-XX-XX) + + - It is now an error to modify the `plugin-files` setting via a + command-line flag that appears after the first non-flag argument + to any command, including a subcommand to `nix`. For example, + `nix-instantiate default.nix --plugin-files ""` must now become + `nix-instantiate --plugin-files "" default.nix`. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5b8ab3387..f784b5160 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv) unsetenv("DISPLAY"); unsetenv("SSH_ASKPASS"); + /* If we ever use the common args framework, make sure to + remove initPlugins below and initialize settings first. + */ if (argc != 2) throw UsageError("called without required arguments"); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index ff96ee7d5..c43e9ebd2 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -79,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName) hiddenCategories.insert(cat); } +void MixCommonArgs::initialFlagsProcessed() +{ + initPlugins(); + pluginsInited(); +} + + } diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 8e53a7361..31bdf527a 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -7,10 +7,14 @@ namespace nix { //static constexpr auto commonArgsCategory = "Miscellaneous common options"; static constexpr auto loggingCategory = "Logging-related options"; -struct MixCommonArgs : virtual Args +class MixCommonArgs : public virtual Args { + void initialFlagsProcessed() override; +public: string programName; MixCommonArgs(const string & programName); +protected: + virtual void pluginsInited() {} }; struct MixDryRun : virtual Args diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 03294b7fe..2780e0bf5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -253,6 +253,7 @@ void PluginFilesSetting::set(const std::string & str, bool append) void initPlugins() { + assert(!settings.pluginFiles.pluginsLoaded); for (const auto & pluginFile : settings.pluginFiles.get()) { Paths pluginFiles; try { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 9377fe4c0..eb11fd64b 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -60,6 +60,7 @@ void Args::parseCmdline(const Strings & _cmdline) verbosity = lvlError; } + bool argsSeen = false; for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -88,6 +89,10 @@ void Args::parseCmdline(const Strings & _cmdline) throw UsageError("unrecognised flag '%1%'", arg); } else { + if (!argsSeen) { + argsSeen = true; + initialFlagsProcessed(); + } pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) @@ -96,6 +101,9 @@ void Args::parseCmdline(const Strings & _cmdline) } processArgs(pendingArgs, true); + + if (!argsSeen) + initialFlagsProcessed(); } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 88f068087..4721c21df 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -132,6 +132,10 @@ protected: std::set hiddenCategories; + /* Called after all command line flags before the first non-flag + argument (if any) have been processed. */ + virtual void initialFlagsProcessed() {} + public: void addFlag(Flag && flag); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d975cd16d..7b4a53919 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -240,8 +240,6 @@ static void main_nix_build(int argc, char * * argv) myArgs.parseCmdline(args); - initPlugins(); - if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 57189d557..3272c6125 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -196,8 +196,6 @@ static int main_nix_channel(int argc, char ** argv) return true; }); - initPlugins(); - switch (cmd) { case cNone: throw UsageError("no command specified"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index c1769790a..4f953fab4 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -74,8 +74,6 @@ static int main_nix_collect_garbage(int argc, char * * argv) return true; }); - initPlugins(); - auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ad2e06067..5e8cc515b 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -43,8 +43,6 @@ static int main_nix_copy_closure(int argc, char ** argv) return true; }); - initPlugins(); - if (sshHost.empty()) throw UsageError("no host name specified"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106a78fc4..0f10a4cbb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1420,8 +1420,6 @@ static int main_nix_env(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index ea2e85eb0..95903d882 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -149,8 +149,6 @@ static int main_nix_instantiate(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 37191b9e6..e17b38c3c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1067,8 +1067,6 @@ static int main_nix_store(int argc, char * * argv) return true; }); - initPlugins(); - if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 26006167d..2cf2a04c9 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -326,8 +326,6 @@ static int main_nix_daemon(int argc, char * * argv) return true; }); - initPlugins(); - runDaemon(stdio); return 0; diff --git a/src/nix/main.cc b/src/nix/main.cc index 1b68cf15b..b078366fa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -283,8 +283,6 @@ void mainWrapped(int argc, char * * argv) if (completions) return; - initPlugins(); - if (args.showVersion) { printVersion(programName); return; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index a831dcd15..b7da3ea5a 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -171,8 +171,6 @@ static int main_nix_prefetch_url(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (args.size() > 2) throw UsageError("too many arguments"); diff --git a/tests/plugins.sh b/tests/plugins.sh index 50bfaf7e9..e22bf4408 100644 --- a/tests/plugins.sh +++ b/tests/plugins.sh @@ -2,6 +2,6 @@ source common.sh set -o pipefail -res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) +res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] From f6c5b05488c588964f51ce97ad2c297fbca7ce96 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 10:04:47 -0500 Subject: [PATCH 143/725] Respect command registrations in plugins. --- doc/manual/src/release-notes/rl-2.4.md | 1 + src/libutil/args.cc | 4 ++-- src/nix/main.cc | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 26ba70904..f7ab9f6ad 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -5,3 +5,4 @@ to any command, including a subcommand to `nix`. For example, `nix-instantiate default.nix --plugin-files ""` must now become `nix-instantiate --plugin-files "" default.nix`. + - Plugins that add new `nix` subcommands are now actually respected. diff --git a/src/libutil/args.cc b/src/libutil/args.cc index eb11fd64b..75eb19d28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -306,8 +306,8 @@ Strings argvToStrings(int argc, char * * argv) return args; } -MultiCommand::MultiCommand(const Commands & commands) - : commands(commands) +MultiCommand::MultiCommand(const Commands & commands_) + : commands(commands_) { expectArgs({ .label = "subcommand", diff --git a/src/nix/main.cc b/src/nix/main.cc index b078366fa..06e221682 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -159,6 +159,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs #include "nix.md" ; } + + // Plugins may add new subcommands. + void pluginsInited() override + { + commands = RegisterCommand::getCommandsFor({}); + } }; static void showHelp(std::vector subcommand) From 1130b2882415b003f5ba2fc0b5466b573fe1b05a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 24 Feb 2021 20:52:22 -0500 Subject: [PATCH 144/725] distributed builds: load remote builder host key from the machines file This is already used by Hydra, and is very useful when materializing a remote builder list from service discovery. This allows the service discovery tool to only sync one file instead of two. --- .../src/advanced-topics/distributed-builds.md | 10 +++++++--- src/libstore/legacy-ssh-store.cc | 2 ++ src/libstore/machines.cc | 6 ++++++ src/libstore/ssh-store.cc | 2 ++ src/libstore/ssh.cc | 16 ++++++++++++++-- src/libstore/ssh.hh | 3 ++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index c6966a50b..580b36736 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -37,7 +37,7 @@ then you need to ensure that the `PATH` of non-interactive login shells contains Nix. > **Warning** -> +> > If you are building via the Nix daemon, it is the Nix daemon user > account (that is, `root`) that should have SSH access to the remote > machine. If you can’t or don’t want to configure `root` to be able to @@ -52,7 +52,7 @@ example, the following command allows you to build a derivation for ```console $ uname Linux - + $ nix build \ '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ --builders 'ssh://mac x86_64-darwin' @@ -103,7 +103,7 @@ default, set it to `-`. ```nix requiredSystemFeatures = [ "kvm" ]; ``` - + will cause the build to be performed on a machine that has the `kvm` feature. @@ -112,6 +112,10 @@ default, set it to `-`. features appear in the derivation’s `requiredSystemFeatures` attribute.. +8. The (base64-encoded) public host key of the remote machine. If omitted, SSH + will use its regular known-hosts file. Specifically, the field is calculated + via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. + For example, the machine specification nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 253c0033e..99b0bb5a8 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -15,6 +15,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig using StoreConfig::StoreConfig; const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; @@ -59,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor , master( host, sshKey, + sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, compress, diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 7db2556f4..b42e5e434 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -54,9 +54,15 @@ ref Machine::openStore() const { if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; storeParams["log-fd"] = "4"; + } + + if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { if (sshKey != "") storeParams["ssh-key"] = sshKey; + if (sshPublicHostKey != "") + storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; } + { auto & fs = storeParams["system-features"]; auto append = [&](auto feats) { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 17c258201..f2caf2aeb 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig using RemoteStoreConfig::RemoteStoreConfig; const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; @@ -34,6 +35,7 @@ public: , master( host, sshKey, + sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, compress) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 84548a6e4..235eed37a 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,24 +2,37 @@ namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) +SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) : host(host) , fakeSSH(host == "localhost") , keyFile(keyFile) + , sshPublicHostKey(sshPublicHostKey) , useMaster(useMaster && !fakeSSH) , compress(compress) , logFD(logFD) { if (host == "" || hasPrefix(host, "-")) throw Error("invalid SSH host name '%s'", host); + + auto state(state_.lock()); + state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); } void SSHMaster::addCommonSSHOpts(Strings & args) { + auto state(state_.lock()); + for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) args.push_back(i); if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); + if (!sshPublicHostKey.empty()) { + Path fileName = (Path) *state->tmpDir + "/host-key"; + auto p = host.rfind("@"); + string thost = p != string::npos ? string(host, p + 1) : host; + writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n"); + args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName}); + } if (compress) args.push_back("-C"); } @@ -87,7 +100,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 4f0f0bd29..dabbcedda 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -12,6 +12,7 @@ private: const std::string host; bool fakeSSH; const std::string keyFile; + const std::string sshPublicHostKey; const bool useMaster; const bool compress; const int logFD; @@ -29,7 +30,7 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); + SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); struct Connection { From 2e199673a523fa81de31ffdd2a25976ce0814631 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 19:43:53 +0100 Subject: [PATCH 145/725] Use `RealisedPath`s in `copyPaths` That way we can copy the realisations too (in addition to the store paths themselves) --- src/libstore/store-api.cc | 31 ++++++++++++++---------- src/libstore/store-api.hh | 9 +++---- src/nix-copy-closure/nix-copy-closure.cc | 6 ++--- src/nix/copy.cc | 13 +++++----- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2658f7617..529c34de5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -783,6 +783,24 @@ void copyStorePath(ref srcStore, ref dstStore, } +std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + StorePathSet storePaths; + std::set realisations; + for (auto path : paths) { + storePaths.insert(path.path()); + if (auto realisation = std::get_if(&path.raw)) + realisations.insert(*realisation); + } + auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + + return pathsMap; +} + std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { @@ -796,7 +814,6 @@ std::map copyPaths(ref srcStore, ref dstStor for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - if (missing.empty()) return pathsMap; Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); @@ -871,21 +888,9 @@ std::map copyPaths(ref srcStore, ref dstStor nrDone++; showProgress(); }); - return pathsMap; } - -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, - SubstituteFlag substitute) -{ - StorePathSet closure; - srcStore->computeFSClosure(storePaths, closure); - copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); -} - - std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven) { std::string path; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6dcd43ed1..63b26422a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,15 +752,12 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const StorePathSet & storePaths, + const RealisedPath::Set&, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); - - -/* Copy the closure of the specified paths from one store to another. */ -void copyClosure(ref srcStore, ref dstStore, - const StorePathSet & storePaths, +std::map copyPaths(ref srcStore, ref dstStore, + const StorePathSet& paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 5e8cc515b..02ccbe541 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -50,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv) auto to = toMode ? openStore(remoteUri) : openStore(); auto from = toMode ? openStore() : openStore(remoteUri); - StorePathSet storePaths2; + RealisedPath::Set storePaths2; for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - StorePathSet closure; - from->computeFSClosure(storePaths2, closure, false, includeOutputs); + RealisedPath::Set closure; + RealisedPath::closure(*from, storePaths2, closure); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c56a1def1..f59f7c76b 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : StorePathsCommand +struct CmdCopy : RealisedPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; - using StorePathsCommand::run; + using RealisedPathsCommand::run; CmdCopy() - : StorePathsCommand(true) + : RealisedPathsCommand(true) { addFlag({ .longName = "from", @@ -75,14 +75,15 @@ struct CmdCopy : StorePathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - StorePathsCommand::run(store); + RealisedPathsCommand::run(store); } - void run(ref srcStore, StorePaths storePaths) override + void run(ref srcStore, std::vector paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), + copyPaths( + srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), NoRepair, checkSigs, substitute); } }; From aead35531a0630b19e41348e103b2d105e2d8dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:37:05 +0100 Subject: [PATCH 146/725] Add a test for the copy of CA paths --- tests/local.mk | 1 + tests/nix-copy-content-addressed.sh | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 tests/nix-copy-content-addressed.sh diff --git a/tests/local.mk b/tests/local.mk index 06be8cec1..a504e397e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,6 +38,7 @@ nix_tests = \ describe-stores.sh \ flakes.sh \ content-addressed.sh \ + nix-copy-content-addressed.sh \ build.sh \ compute-levels.sh # parallel.sh diff --git a/tests/nix-copy-content-addressed.sh b/tests/nix-copy-content-addressed.sh new file mode 100755 index 000000000..2e0dea2d2 --- /dev/null +++ b/tests/nix-copy-content-addressed.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +source common.sh + +# Globally enable the ca derivations experimental flag +sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +ensureCorrectlyCopied () { + attrPath="$1" + nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath" +} + +testOneCopy () { + clearStore + rm -rf "$REMOTE_STORE_DIR" + + attrPath="$1" + nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + + ensureCorrectlyCopied "$attrPath" + + # Ensure that we can copy back what we put in the store + clearStore + nix copy --from $REMOTE_STORE \ + --file ./content-addressed.nix "$attrPath" \ + --no-check-sigs +} + +for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do + testOneCopy "$attrPath" +done From f67ff1f5756018387a2d23c8f6772580192d30ad Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 17:58:28 +0100 Subject: [PATCH 147/725] Don't crash when copying realisations to a non-ca remote Rather throw a proper exception, and catch&log it on the client side --- src/libstore/globals.cc | 7 ++++++- src/libstore/globals.hh | 18 ++++++++++++++---- src/libstore/local-store.cc | 1 + src/libstore/store-api.cc | 14 ++++++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2780e0bf5..8d44003f4 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -165,10 +165,15 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) return std::find(f.begin(), f.end(), name) != f.end(); } +MissingExperimentalFeature::MissingExperimentalFeature(std::string feature) + : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature) + , missingFeature(feature) + {} + void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); + throw MissingExperimentalFeature(name); } bool Settings::isWSL1() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index df61d6417..25351f55c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,6 +45,16 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +/* MakeError(MissingExperimentalFeature, Error); */ +class MissingExperimentalFeature: public Error +{ +public: + std::string missingFeature; + + MissingExperimentalFeature(std::string feature); + virtual const char* sname() const override { return "MissingExperimentalFeature"; } +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -632,7 +642,7 @@ public: is `root`. > **Warning** - > + > > Adding a user to `trusted-users` is essentially equivalent to > giving that user root access to the system. For example, the user > can set `sandbox-paths` and thereby obtain read access to @@ -722,13 +732,13 @@ public: The program executes with no arguments. The program's environment contains the following environment variables: - - `DRV_PATH` + - `DRV_PATH` The derivation for the built paths. Example: `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - `OUT_PATHS` + - `OUT_PATHS` Output paths of the built derivation, separated by a space character. @@ -759,7 +769,7 @@ public: documentation](https://ec.haxx.se/usingcurl-netrc.html). > **Note** - > + > > This must be an absolute path, and `~` is not resolved. For > example, `~/.netrc` won't resolve to your home directory's > `.netrc`. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0962418dd..90fb4a4bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -655,6 +655,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { + settings.requireExperimentalFeature("ca-derivations"); auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 529c34de5..ac1d8ee2c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -794,8 +794,18 @@ std::map copyPaths(ref srcStore, ref dstStor realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); - for (auto& realisation : realisations) { - dstStore->registerDrvOutput(realisation); + try { + for (auto& realisation : realisations) { + dstStore->registerDrvOutput(realisation); + } + } catch (MissingExperimentalFeature & e) { + // Don't fail if the remote doesn't support CA derivations is it might + // not be whithin our control to change that, and we might still want + // to at least copy the output paths. + if (e.missingFeature == "ca-derivations") + ignoreException(); + else + throw; } return pathsMap; From 3b76f8f252c12fbeb49aa2f6f695b4622e9fcc5d Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 19 Feb 2021 18:02:26 +0100 Subject: [PATCH 148/725] Ensure that the ca-derivations bit is set when copying realisations This should already hold, but better ensure it for future-proof-nees --- src/libstore/store-api.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ac1d8ee2c..db84ec7a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,6 +786,7 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { + settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto path : paths) { From c182aac98ab6548c16b6686638591ba5b034026a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 25 Feb 2021 17:10:45 +0100 Subject: [PATCH 149/725] Apply @edolstra stylistic suggestions Mostly removing useless comments and adding spaces before `&` Co-authored-by: Eelco Dolstra --- src/libstore/globals.hh | 1 - src/libstore/store-api.cc | 6 +++--- src/libstore/store-api.hh | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 25351f55c..a51d9c2f1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -45,7 +45,6 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -/* MakeError(MissingExperimentalFeature, Error); */ class MissingExperimentalFeature: public Error { public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index db84ec7a2..b7a3f7b11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -789,19 +789,19 @@ std::map copyPaths(ref srcStore, ref dstStor settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; - for (auto path : paths) { + for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) realisations.insert(*realisation); } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - for (auto& realisation : realisations) { + for (auto & realisation : realisations) { dstStore->registerDrvOutput(realisation); } } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might - // not be whithin our control to change that, and we might still want + // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == "ca-derivations") ignoreException(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 63b26422a..742cd18db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -752,7 +752,7 @@ void copyStorePath(ref srcStore, ref dstStore, that. Returns a map of what each path was copied to the dstStore as. */ std::map copyPaths(ref srcStore, ref dstStore, - const RealisedPath::Set&, + const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); From c43f446f4e3a1a8d91560b6ebbcc7d4fbbbf71c4 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 25 Feb 2021 16:58:27 +0100 Subject: [PATCH 150/725] Make `nix copy` work without the ca-derivations flag The experimental feature was by mistake required for `nix copy` to work at oll --- src/libstore/store-api.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b7a3f7b11..77c310988 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -786,13 +786,14 @@ void copyStorePath(ref srcStore, ref dstStore, std::map copyPaths(ref srcStore, ref dstStore, const RealisedPath::Set & paths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - settings.requireExperimentalFeature("ca-derivations"); StorePathSet storePaths; std::set realisations; for (auto & path : paths) { storePaths.insert(path.path()); - if (auto realisation = std::get_if(&path.raw)) + if (auto realisation = std::get_if(&path.raw)) { + settings.requireExperimentalFeature("ca-derivations"); realisations.insert(*realisation); + } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { From 20ea1de77d9210e145d5ebb1dccd34c856149b2c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 12:35:29 +0100 Subject: [PATCH 151/725] Use std::make_unique --- src/libstore/build/derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 804a79e4c..33c3aeb6e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -259,10 +259,8 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); - auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); - /* Get the derivation. */ - drv = std::unique_ptr(fullDrv); + drv = std::make_unique(worker.store.derivationFromPath(drvPath)); haveDerivation(); } From 453c3a603f4e6fa3f8c706e73f9869bc7f76c640 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 14:55:54 +0100 Subject: [PATCH 152/725] nix flake update: Recreate the lock file This is probably what most people expect it to do. Fixes #3781. There is a new command 'nix flake lock' that has the old behaviour of 'nix flake update', i.e. it just adds missing lock file entries unless overriden using --update-input. --- src/libutil/args.cc | 8 ++++++++ src/libutil/args.hh | 2 ++ src/nix/flake-lock.md | 38 ++++++++++++++++++++++++++++++++++++++ src/nix/flake-update.md | 37 +++++++++---------------------------- src/nix/flake.cc | 34 +++++++++++++++++++++++++++++++++- tests/flakes.sh | 28 ++++++++++++++-------------- 6 files changed, 104 insertions(+), 43 deletions(-) create mode 100644 src/nix/flake-lock.md diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 75eb19d28..afed0670f 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -19,6 +19,14 @@ void Args::addFlag(Flag && flag_) if (flag->shortName) shortFlags[flag->shortName] = flag; } +void Args::removeFlag(const std::string & longName) +{ + auto flag = longFlags.find(longName); + assert(flag != longFlags.end()); + if (flag->second->shortName) shortFlags.erase(flag->second->shortName); + longFlags.erase(flag); +} + void Completions::add(std::string completion, std::string description) { assert(description.find('\n') == std::string::npos); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 4721c21df..c08ba8abd 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -140,6 +140,8 @@ public: void addFlag(Flag && flag); + void removeFlag(const std::string & longName); + void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md new file mode 100644 index 000000000..2af0ad81e --- /dev/null +++ b/src/nix/flake-lock.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake lock --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated unless +required by a flag such as `--update-input`. + +Note that every command that operates on a flake will also update the +lock file if needed, and supports the same flags. Therefore, + +```console +# nix flake lock --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index a2ffedd2a..03b50e38e 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,52 +2,33 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: - - ```console - # nix flake update --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - ``` - * Recreate the lock file (i.e. update all inputs) and commit the new lock file: ```console - # nix flake update --recreate-lock-file --commit-lock-file + # nix flake update + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Note that every command that operates on a flake will -also update the lock file if needed, and supports the same -flags. Therefore, +This command recreates the lock file of a flake (`flake.lock`), thus +updating the lock for every mutable input (like `nixpkgs`) to its +current version. This is equivalent to passing `--recreate-lock-file` +to any command that operates on a flake. That is, ```console -# nix flake update --update-input nixpkgs +# nix flake update # nix build ``` is equivalent to: ```console -# nix build --update-input nixpkgs +# nix build --recreate-lock-file ``` -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. - -> **Note** -> -> This command does *not* update locks that are already present unless -> you explicitly ask for it using `--update-input` or -> `--recreate-lock-file`. Thus, if the lock file already has locks for -> every input, then `nix flake update` (without arguments) does -> nothing. - )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b9cde5d6d..2f0c468a8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -104,6 +104,14 @@ struct CmdFlakeUpdate : FlakeCommand return "update flake lock file"; } + CmdFlakeUpdate() + { + /* Remove flags that don't make sense. */ + removeFlag("recreate-lock-file"); + removeFlag("update-input"); + removeFlag("no-update-lock-file"); + } + std::string doc() override { return @@ -113,7 +121,30 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { - /* Use --refresh by default for 'nix flake update'. */ + settings.tarballTtl = 0; + + lockFlags.recreateLockFile = true; + + lockFlake(); + } +}; + +struct CmdFlakeLock : FlakeCommand +{ + std::string description() override + { + return "create missing lock file entries"; + } + + std::string doc() override + { + return + #include "flake-lock.md" + ; + } + + void run(nix::ref store) override + { settings.tarballTtl = 0; lockFlake(); @@ -1006,6 +1037,7 @@ struct CmdFlake : NixMultiCommand CmdFlake() : MultiCommand({ {"update", []() { return make_ref(); }}, + {"lock", []() { return make_ref(); }}, {"info", []() { return make_ref(); }}, {"list-inputs", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, diff --git a/tests/flakes.sh b/tests/flakes.sh index 2b7bcdd68..25ba2ac43 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -232,7 +232,7 @@ nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2 nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh # Updating the flake should not change the lockfile. -nix flake update $flake2Dir +nix flake lock $flake2Dir [[ -z $(git -C $flake2Dir diff master) ]] # Now we should be able to build the flake in pure mode. @@ -354,10 +354,10 @@ nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord nix build -o $TEST_ROOT/result flake4#xyzzy # Test 'nix flake update' and --override-flake. -nix flake update $flake3Dir +nix flake lock $flake3Dir [[ -z $(git -C $flake3Dir diff master) ]] -nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs +nix flake update $flake3Dir --override-flake flake2 nixpkgs [[ ! -z $(git -C $flake3Dir diff master) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore @@ -389,7 +389,7 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Fri, 26 Feb 2021 16:03:39 +0100 Subject: [PATCH 153/725] flake.lock: Update Flake input changes: * Updated 'nixpkgs': 'github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31' -> 'github:NixOS/nixpkgs/0e499fde7af3c28d63e9b13636716b86c3162b93' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 6fe52fbfd..9867e694b 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1602702596, - "narHash": "sha256-fqJ4UgOb4ZUnCDIapDb4gCrtAah5Rnr2/At3IzMitig=", + "lastModified": 1614309161, + "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ad0d20345219790533ebe06571f82ed6b034db31", + "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93", "type": "github" }, "original": { From 14f51880bad5145e73eb150797e757440925913b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:29:30 +0100 Subject: [PATCH 154/725] Update src/build-remote/build-remote.cc --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 228aba35a..1be491603 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -288,7 +288,7 @@ connected: auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { - notice("Missing output %s", outputName); + debug("missing output %s", outputName); assert(result.builtOutputs.count(thisOutputId)); auto newRealisation = result.builtOutputs.at(thisOutputId); missingRealisations.insert(newRealisation); From 17c98e03eac45b3c298567e8a1c04e3d4c4aa0d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:29:37 +0100 Subject: [PATCH 155/725] Update src/build-remote/build-remote.cc --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1be491603..7f3636f6b 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,7 +311,7 @@ connected: localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ copyPaths(ref(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } - // XXX: Should e done as part of `copyPaths` + // XXX: Should be done as part of `copyPaths` for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty From 076d2b04da72607b67e581572a31db2a220589ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2021 16:30:12 +0100 Subject: [PATCH 156/725] Update src/libstore/build/derivation-goal.cc --- src/libstore/build/derivation-goal.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6052b625d..a5622f990 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1164,7 +1164,6 @@ HookReply DerivationGoal::tryBuildHook() // XXX: Does this include known CA outputs? if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); - /* missingPaths.insert(status.known->path); */ } worker_proto::write(worker.store, hook->sink, missingOutputs); } From f54976d77bd144535e9b4844dbdb6bc52eac11fd Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 26 Feb 2021 16:34:33 +0100 Subject: [PATCH 157/725] Simplify the case where the drv is a purely input-addressed one --- src/build-remote/build-remote.cc | 2 +- src/libstore/build/entry-points.cc | 2 +- src/libstore/derivations.cc | 11 +++++++++++ src/libstore/derivations.hh | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 7f3636f6b..736b81542 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -283,7 +283,7 @@ connected: std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 99b3fa070..3a05a022c 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -69,7 +69,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat outputId, Realisation{ outputId, *staticOutput.second} ); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { auto realisation = this->queryRealisation(outputId); if (realisation) result.builtOutputs.insert_or_assign( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6d0742b4f..fe98182bb 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -57,6 +57,17 @@ bool derivationIsFixed(DerivationType dt) { assert(false); } +bool derivationHasKnownOutputPaths(DerivationType dt) { + switch (dt) { + case DerivationType::InputAddressed: return true; + case DerivationType::CAFixed: return true; + case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; + }; + assert(false); +} + + bool derivationIsImpure(DerivationType dt) { switch (dt) { case DerivationType::InputAddressed: return false; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4e5985fab..061d70f69 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -94,6 +94,11 @@ bool derivationIsFixed(DerivationType); derivation is controlled separately. Never true for non-CA derivations. */ bool derivationIsImpure(DerivationType); +/* Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the closure. + */ +bool derivationHasKnownOutputPaths(DerivationType); + struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ From 05cc5a858717c092e1835e2b0fec4c4b1a7fc97e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Feb 2021 06:26:35 +0000 Subject: [PATCH 158/725] Copy {,local-}derivation-goal.{cc,h} Doing this prior to splitting, so we get better diff with default options (e.g. on GitHub). --- src/libstore/build/local-derivation-goal.cc | 3902 +++++++++++++++++++ src/libstore/build/local-derivation-goal.hh | 373 ++ 2 files changed, 4275 insertions(+) create mode 100644 src/libstore/build/local-derivation-goal.cc create mode 100644 src/libstore/build/local-derivation-goal.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc new file mode 100644 index 000000000..924c69fb7 --- /dev/null +++ b/src/libstore/build/local-derivation-goal.cc @@ -0,0 +1,3902 @@ +#include "derivation-goal.hh" +#include "hook-instance.hh" +#include "worker.hh" +#include "builtins.hh" +#include "builtins/buildenv.hh" +#include "references.hh" +#include "finally.hh" +#include "util.hh" +#include "archive.hh" +#include "json.hh" +#include "compression.hh" +#include "daemon.hh" +#include "worker-protocol.hh" +#include "topo-sort.hh" +#include "callback.hh" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_STATVFS +#include +#endif + +/* Includes required for chroot support. */ +#if __linux__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_SECCOMP +#include +#endif +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +#endif + +#if __APPLE__ +#include +#include +#endif + +#include +#include + +#include + +namespace nix { + +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: wrap errors. + ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); + logError(ei); + } + } +} + +const Path DerivationGoal::homeDir = "/homeless-shelter"; + +DerivationGoal::DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(true) + , drvPath(drvPath) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &DerivationGoal::getDerivation; + name = fmt( + "building of '%s' from .drv file", + StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); +} + + +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(false) + , drvPath(drvPath) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + this->drv = std::make_unique(drv); + + state = &DerivationGoal::haveDerivation; + name = fmt( + "building of '%s' from in-memory derivation", + StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(this->drvPath); +} + + +DerivationGoal::~DerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { killChild(); } catch (...) { ignoreException(); } + try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreException(); } +} + + +string DerivationGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); +} + + +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + +void DerivationGoal::killChild() +{ + if (pid != -1) { + worker.childTerminated(this); + + if (buildUser) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser->kill(); + pid.wait(); + } else + pid.kill(); + + assert(pid == -1); + } + + hook.reset(); +} + + +void DerivationGoal::timedOut(Error && ex) +{ + killChild(); + done(BuildResult::TimedOut, ex); +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + for (auto & i : outputs) + if (wantedOutputs.insert(i).second) + needRestart = true; +} + + +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); + + state = &DerivationGoal::loadDerivation; +} + + +void DerivationGoal::loadDerivation() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = std::make_unique(worker.store.derivationFromPath(drvPath)); + + haveDerivation(); +} + + +void DerivationGoal::haveDerivation() +{ + trace("have derivation"); + + if (drv->type() == DerivationType::CAFloating) + settings.requireExperimentalFeature("ca-derivations"); + + retrySubstitution = false; + + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + worker.store.addTempRoot(*i.second.second); + + auto outputHashes = staticOutputHashes(worker.store, *drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + + /* Check what outputs paths are not already valid. */ + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + + /* If they are all valid, then we're done. */ + if (allValid && buildMode == bmNormal) { + done(BuildResult::AlreadyValid); + return; + } + + parsedDrv = std::make_unique(drvPath, *drv); + + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known) { + warn("do not know how to query for unknown floating content-addressed derivation output yet"); + /* Nothing to wait for; tail call */ + return DerivationGoal::gaveUpOnSubstitution(); + } + addWaitee(upcast_goal(worker.makeSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv)))); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstitutionTried(); + else + state = &DerivationGoal::outputsSubstitutionTried; +} + + +void DerivationGoal::outputsSubstitutionTried() +{ + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { + done(BuildResult::TransientFailure, + fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); + return; + } + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + checkPathValidity(); + size_t nrInvalid = 0; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) + nrInvalid++; + } + + if (buildMode == bmNormal && nrInvalid == 0) { + done(BuildResult::Substituted); + return; + } + if (buildMode == bmRepair && nrInvalid == 0) { + repairClosure(); + return; + } + if (buildMode == bmCheck && nrInvalid > 0) + throw Error("some outputs of '%s' are not valid, so checking is not possible", + worker.store.printStorePath(drvPath)); + + /* Nothing to wait for; tail call */ + gaveUpOnSubstitution(); +} + +/* At least one of the output paths could not be + produced using a substitute. So we have to build instead. */ +void DerivationGoal::gaveUpOnSubstitution() +{ + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs.clear(); + + /* The inputs must be built before we can build this goal. */ + if (useDerivation) + for (auto & i : dynamic_cast(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + + for (auto & i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + auto outputs = queryDerivationOutputMap(); + StorePathSet outputClosure; + for (auto & i : outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second, outputClosure); + } + + /* Filter out our own outputs (which we have already checked). */ + for (auto & i : outputs) + outputClosure.erase(i.second); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + StorePathSet inputClosure; + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); + std::map outputsToDrv; + for (auto & i : inputClosure) + if (i.isDerivation()) { + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + for (auto & j : depOutputs) + if (j.second) + outputsToDrv.insert_or_assign(*j.second, i); + } + + /* Check each path (slow!). */ + for (auto & i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printError( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + auto drvPath2 = outputsToDrv.find(i); + if (drvPath2 == outputsToDrv.end()) + addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); + else + addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + } + + if (waitees.empty()) { + done(BuildResult::AlreadyValid); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error("some paths in the output closure of derivation '%s' could not be repaired", + worker.store.printStorePath(drvPath)); + done(BuildResult::AlreadyValid); +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + if (!useDerivation) + throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + if (useDerivation) { + auto & fullDrv = *dynamic_cast(drv.get()); + + if (settings.isExperimentalFeatureEnabled("ca-derivations") && + ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed)) { + /* We are be able to resolve this derivation based on the + now-known results of dependencies. If so, we become a stub goal + aliasing that resolved derivation goal */ + std::optional attempt = fullDrv.tryResolve(worker.store); + assert(attempt); + Derivation drvResolved { *std::move(attempt) }; + + auto pathResolved = writeDerivation(worker.store, drvResolved); + resolvedDrv = drvResolved; + + auto msg = fmt("Resolved derivation: '%s' -> '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved)); + act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, + Logger::Fields { + worker.store.printStorePath(drvPath), + worker.store.printStorePath(pathResolved), + }); + + auto resolvedGoal = worker.makeDerivationGoal( + pathResolved, wantedOutputs, buildMode); + addWaitee(resolvedGoal); + + state = &DerivationGoal::resolvedFinished; + return; + } + + for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(drvPath)); + auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + for (auto & j : wantedDepOutputs) { + if (outputs.count(j) > 0) { + auto optRealizedInput = outputs.at(j); + if (!optRealizedInput) + throw Error( + "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.computeFSClosure(*optRealizedInput, inputPaths); + } else + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + } + } + + /* Second, the input sources. */ + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + + debug("added input paths %s", worker.store.showPaths(inputPaths)); + + /* What type of derivation are we building? */ + derivationType = drv->type(); + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + + result = BuildResult(); +} + +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); +} + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + /* Obtain locks on all output paths, if the paths are known a priori. + + The locks are automatically released when we exit this function or Nix + crashes. If we can't acquire the lock, then continue; hopefully some + other goal can start a build, and if not, the main loop will sleep a few + seconds and then retry this goal. */ + PathSet lockFiles; + /* FIXME: Should lock something like the drv itself so we don't build same + CA drv concurrently */ + if (dynamic_cast(&worker.store)) + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + + if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); + worker.waitForAWhile(shared_from_this()); + return; + } + + actLock.reset(); + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + checkPathValidity(); + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + if (buildMode != bmCheck && allValid) { + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); + outputLocks.setDeletion(true); + done(BuildResult::AlreadyValid); + return; + } + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto & [_, status] : initialOutputs) { + if (!status.known || status.known->isValid()) continue; + auto storePath = status.known->path; + debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); + deletePath(worker.store.Store::toRealPath(storePath)); + } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + actLock.reset(); + result.startTime = time(0); // inexact + state = &DerivationGoal::buildDone; + started(); + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + actLock.reset(); + + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + /* Make sure that we are allowed to start a build. */ + if (!dynamic_cast(&worker.store)) { + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + } + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + actLock.reset(); + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + done(BuildResult::InputRejected, e); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; + + started(); +} + + +static void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + + +/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if + it's a directory and we're not root (to be able to update the + directory's parent link ".."). */ +static void movePath(const Path & src, const Path & dst) +{ + auto st = lstat(src); + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + +void replaceValidPath(const Path & storePath, const Path & tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + if (pathExists(storePath)) + movePath(storePath, oldPath); + + try { + movePath(tmpPath, storePath); + } catch (...) { + try { + // attempt to recover + movePath(oldPath, storePath); + } catch (...) { + ignoreException(); + } + throw; + } + + deletePath(oldPath); +} + + +MakeError(NotDeterministic, BuildError); + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + sandboxMountNamespace = -1; + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill() : pid.kill(); + + debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); + + result.timesBuilt++; + result.stopTime = time(0); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; + } else + builderOut.readSide = -1; + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + + bool diskFull = false; + + try { + + /* Check the exit status. */ + if (!statusOk(status)) { + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + if (auto localStore = dynamic_cast(&worker.store)) { + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } + + auto msg = fmt("builder for '%s' %s", + yellowtxt(worker.store.printStorePath(drvPath)), + statusToString(status)); + + if (!logger->isVerbose() && !logTail.empty()) { + msg += fmt(";\nlast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } + msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + worker.store.printStorePath(drvPath)); + } + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + registerOutputs(); + + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{worker.store.printStorePath(drvPath)}); + PushActivity pact(act.id); + StorePathSet outputPaths; + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); + } + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (std::string_view data) override { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + + if (buildMode == bmCheck) { + deleteTmpDir(true); + done(BuildResult::Built); + return; + } + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(worker.store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError & e) { + outputLocks.unlock(); + + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; + + else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { + } + + else { + st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; + } + + done(st, e); + return; + } + + done(BuildResult::Built); +} + +void DerivationGoal::resolvedFinished() { + assert(resolvedDrv); + + auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv->outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(initialOutputs.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = worker.store.queryRealisation( + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + ); + // We've just built it, but maybe the build failed, in which case the + // realisation won't be there + if (realisation) { + auto newRealisation = *realisation; + newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; + worker.store.registerDrvOutput(newRealisation); + } else { + // If we don't have a realisation, then it must mean that something + // failed when building the resolved drv + assert(!result.success()); + } + } + + // This is potentially a bit fishy in terms of error reporting. Not sure + // how to do it in a cleaner way + amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); +} + +HookReply DerivationGoal::tryBuildHook() +{ + if (!worker.tryBuildHook || !useDerivation) return rpDecline; + + if (!worker.hook) + worker.hook = std::make_unique(); + + try { + + /* Send the request to the hook. */ + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << worker.store.printStorePath(drvPath) + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + auto s = [&]() { + try { + return readLine(worker.hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response from the build hook"); + throw e; + } + }(); + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + else { + s += "\n"; + writeToStderr(s); + } + } + + debug("hook reply is '%1%'", reply); + + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; + else if (reply != "accept") + throw Error("bad hook reply '%s'", reply); + + } catch (SysError & e) { + if (e.errNo == EPIPE) { + printError( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); + worker.hook = 0; + return rpDecline; + } else + throw; + } + + hook = std::move(worker.hook); + + try { + machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw e; + } + + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + worker_proto::write(worker.store, hook->sink, inputPaths); + + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + { + StringSet missingOutputs; + for (auto & [outputName, status] : initialOutputs) { + // XXX: Does this include known CA outputs? + if (buildMode != bmCheck && status.known && status.known->isValid()) continue; + missingOutputs.insert(outputName); + } + worker_proto::write(worker.store, hook->sink, missingOutputs); + } + + hook->sink = FdSink(); + hook->toHook.writeSide = -1; + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + set fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); + + return rpAccept; +} + + +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->runChild(); + return 1; +} + + +StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) +{ + StorePathSet paths; + + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); + + worker.store.computeFSClosure({storePath}, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = paths; + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputsAndOptPaths(worker.store)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + worker.store.computeFSClosure(*k.second.second, paths); + } + } + } + + return paths; +} + +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } + }); +} + + +void linkOrCopy(const Path & from, const Path & to) +{ + if (link(from.c_str(), to.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum link count on a + file (e.g. 32000 of ext3), which is quite possible after a + 'nix-store --optimise'. FIXME: actually, why don't we just + bind-mount in this case? + + It can also fail with EPERM in BeegFS v7 and earlier versions + which don't allow hard-links to other directories */ + if (errno != EMLINK && errno != EPERM) + throw SysError("linking '%s' to '%s'", to, from); + copyPath(from, to); + } +} + + +void DerivationGoal::startBuilder() +{ + /* Right platform? */ + if (!parsedDrv->canBuildLocally(worker.store)) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + worker.store.printStorePath(drvPath), + settings.thisSystem, + concatStringsSep(", ", worker.store.systemFeatures)); + + if (drv->isBuiltin()) + preloadNSS(); + +#if __APPLE__ + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); +#endif + + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = !(derivationIsImpure(derivationType)) && !noChroot; + } + + if (auto localStoreP = dynamic_cast(&worker.store)) { + auto & localStore = *localStoreP; + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + } + + /* Create a temporary directory where the build will take + place. */ + tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); + + chownToBuilder(tmpDir); + + for (auto & [outputName, status] : initialOutputs) { + /* Set scratch path we'll actually use during the build. + + If we're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + auto scratchPath = + !status.known + ? makeFallbackPath(outputName) + : !needsHashRewrite() + /* Can always use original path in sandbox */ + ? status.known->path + : !status.known->isPresent() + /* If path doesn't yet exist can just use it */ + ? status.known->path + : buildMode != bmRepair && !status.known->isValid() + /* If we aren't repairing we'll delete a corrupted path, so we + can use original path */ + ? status.known->path + : /* If we are repairing or the path is totally valid, we'll need + to use a temporary path */ + makeFallbackPath(status.known->path); + scratchOutputs.insert_or_assign(outputName, scratchPath); + + /* A non-removed corrupted path needs to be stored here, too */ + if (buildMode == bmRepair && !status.known->isValid()) + redirectedBadOutputs.insert(status.known->path); + + /* Substitute output placeholders with the scratch output paths. + We'll use during the build. */ + inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); + + /* Additional tasks if we know the final path a priori. */ + if (!status.known) continue; + auto fixedFinalPath = status.known->path; + + /* Additional tasks if the final and scratch are both known and + differ. */ + if (fixedFinalPath == scratchPath) continue; + + /* Ensure scratch path is ours to use. */ + deletePath(worker.store.printStorePath(scratchPath)); + + /* Rewrite and unrewrite paths */ + { + std::string h1 { fixedFinalPath.hashPart() }; + std::string h2 { scratchPath.hashPart() }; + inputRewrites[h1] = h2; + } + + redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); + } + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph").value_or(""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto storePathS = *i++; + if (!worker.store.isInStore(storePathS)) + throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + auto storePath = worker.store.toStorePath(storePathS).first; + + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences({storePath}), false, false)); + } + } + + if (useChroot) { + + /* Allow a user-configurable set of directories from the + host file system. */ + dirsInChroot.clear(); + + for (auto i : settings.sandboxPaths.get()) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == string::npos) + dirsInChroot[i] = {i, optional}; + else + dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + } + dirsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + StorePathSet closure; + for (auto & i : dirsInChroot) + try { + if (worker.store.isInStore(i.second.source)) + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); + } catch (InvalidPath & e) { + } catch (Error & e) { + e.addTrace({}, "while processing 'sandbox-paths'"); + throw; + } + for (auto & i : closure) { + auto p = worker.store.printStorePath(i); + dirsInChroot.insert_or_assign(p, p); + } + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + worker.store.printStorePath(drvPath), i); + + dirsInChroot[i] = i; + } + +#if __linux__ + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared(chrootRootDir); + + printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + + if (mkdir(chrootRootDir.c_str(), 0750) == -1) + throw SysError("cannot create '%1%'", chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", sandboxGid())); + + /* Create /etc/hosts with localhost entry. */ + if (!(derivationIsImpure(derivationType))) + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); + + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + Path r = worker.store.toRealPath(p); + if (S_ISDIR(lstat(r).st_mode)) + dirsInChroot.insert_or_assign(p, r); + else + linkOrCopy(r, chrootRootDir + p); + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.dirsInChroot + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + /* If the name isn't known a priori (i.e. floating + content-addressed derivation), the temporary location we use + should be fresh. Freshness means it is impossible that the path + is already in the sandbox, so we don't need to worry about + removing it. */ + if (i.second.second) + dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + } + +#elif __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ +#else + throw Error("sandboxing builds is not supported on this platform"); +#endif + } + + if (needsHashRewrite() && pathExists(homeDir)) + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); + + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, format("executing pre-build hook '%1%'") + % settings.preBuildHook); + auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : + Strings({ worker.store.printStorePath(drvPath) }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos - lastPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error("unknown pre-build hook command '%1%'", line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == string::npos) + dirsInChroot[line] = line; + else + dirsInChroot[string(line, 0, p)] = string(line, p + 1); + } + } + } + } + + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) + startDaemon(); + + /* Run the builder. */ + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } +#if __APPLE__ + else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } +#endif + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + result.startTime = time(0); + + /* Fork a child to build the package. */ + ProcessOptions options; + +#if __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + if (!(derivationIsImpure(derivationType))) + privateNetwork = true; + + userNamespaceSync.create(); + + options.allowVfork = false; + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + static bool userNamespacesEnabled = + pathExists(maxUserNamespaces) + && trim(readFile(maxUserNamespaces)) != "0"; + + usingUserNamespace = userNamespacesEnabled; + + Pid helper = startProcess([&]() { + + /* Drop additional groups here because we can't do it + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ + if (getuid() == 0 && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) + flags |= CLONE_NEWNET; + if (usingUserNamespace) + flags |= CLONE_NEWUSER; + + pid_t child = clone(childEntry, stack + stackSize, flags, this); + if (child == -1 && errno == EINVAL) { + /* Fallback for Linux < 2.13 where CLONE_NEWPID and + CLONE_PARENT are not allowed together. */ + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unprivileged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + usingUserNamespace = false; + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); + if (child == -1) throw SysError("cloning builder process"); + + writeFull(builderOut.writeSide.get(), + fmt("%d %d\n", usingUserNamespace, child)); + _exit(0); + }, options); + + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) + throw Error("unable to start build process"); + + userNamespaceSync.readSide = -1; + + /* Close the write side to prevent runChild() from hanging + reading from this. */ + Finally cleanup([&]() { + userNamespaceSync.writeSide = -1; + }); + + auto ss = tokenizeString>(readLine(builderOut.readSide.get())); + assert(ss.size() == 2); + usingUserNamespace = ss[0] == "1"; + pid = string2Int(ss[1]).value(); + + if (usingUserNamespace) { + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + fmt("%d %d 1", sandboxUid(), hostUid)); + + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + fmt("%d %d 1", sandboxGid(), hostGid)); + } else { + debug("note: not using a user namespace"); + if (!buildUser) + throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); + } + + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + + /* Save the mount namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + + /* Signal the builder that we've updated its user namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + + } else +#endif + { + fallback: + options.allowVfork = !buildUser && !drv->isBuiltin(); + pid = startProcess([&]() { + runChild(); + }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = -1; + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + + /* Check if setting up the build environment failed. */ + std::vector msgs; + while (true) { + string msg = [&]() { + try { + return readLine(builderOut.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + concatStringsSep("|", msgs)); + throw e; + } + }(); + if (string(msg, 0, 1) == "\2") break; + if (string(msg, 0, 1) == "\1") { + FdSource source(builderOut.readSide.get()); + auto ex = readError(source); + ex.addTrace({}, "while setting up the build environment"); + throw ex; + } + debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); + } +} + + +void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + + StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); + for (auto & i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(htSHA256, i.first); + string fn = ".attr-" + hash.to_string(Base32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (derivationIsImpure(derivationType)) { + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i).value_or(""); + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + +void DerivationGoal::writeStructuredAttrs() +{ + auto structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); + } + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(worker.store.parseStorePath(p.get())); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); +} + +struct RestrictedStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; + +/* A wrapper around LocalStore that only allows building/querying of + paths that are in the input closures of the build or were added via + recursive Nix calls. */ +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore +{ + ref next; + + DerivationGoal & goal; + + RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) + { } + + Path getRealStoreDir() override + { return next->realStoreDir; } + + std::string getUri() override + { return next->getUri(); } + + StorePathSet queryAllValidPaths() override + { + StorePathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); + return paths; + } + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override + { + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.reset(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); + }; + + void queryReferrers(const StorePath & path, StorePathSet & referrers) override + { } + + std::map> queryPartialDerivationOutputMap(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); + return next->queryPartialDerivationOutputMap(path); + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { throw Error("queryPathFromHashPart"); } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + { throw Error("addToStore"); } + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override + { + next->addToStore(info, narSource, repair, checkSigs); + goal.addDependency(info.path); + } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override + { + auto path = next->addTextToStore(name, s, references, repair); + goal.addDependency(path); + return path; + } + + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); + LocalFSStore::narFromPath(path, sink); + } + + void ensurePath(const StorePath & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); + /* Nothing to be done; 'path' must already be valid. */ + } + + void registerDrvOutput(const Realisation & info) override + // XXX: This should probably be allowed as a no-op if the realisation + // corresponds to an allowed derivation + { throw Error("registerDrvOutput"); } + + std::optional queryRealisation(const DrvOutput & id) override + // XXX: This should probably be allowed if the realisation corresponds to + // an allowed derivation + { throw Error("queryRealisation"); } + + void buildPaths(const std::vector & paths, BuildMode buildMode) override + { + if (buildMode != bmNormal) throw Error("unsupported build mode"); + + StorePathSet newPaths; + + for (auto & path : paths) { + if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + } + + next->buildPaths(paths, buildMode); + + for (auto & path : paths) { + if (!path.path.isDerivation()) continue; + auto outputs = next->queryDerivationOutputMap(path.path); + for (auto & output : outputs) + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second); + } + + StorePathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + void addTempRoot(const StorePath & path) override + { } + + void addIndirectRoot(const Path & path) override + { } + + Roots findRoots(bool censor) override + { return Roots(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { } + + void addSignatures(const StorePath & storePath, const StringSet & sigs) override + { unsupported("addSignatures"); } + + void queryMissing(const std::vector & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + uint64_t & downloadSize, uint64_t & narSize) override + { + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + std::vector allowed; + for (auto & path : targets) { + if (goal.isAllowed(path.path)) + allowed.emplace_back(path); + else + unknown.insert(path.path); + } + + next->queryMissing(allowed, willBuild, willSubstitute, + unknown, downloadSize, narSize); + } +}; + + +void DerivationGoal::startDaemon() +{ + settings.requireExperimentalFeature("recursive-nix"); + + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = worker.store.storeDir; + if (auto localStore = dynamic_cast(&worker.store)) + params["root"] = localStore->rootDir; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = make_ref(params, + ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR) continue; + if (errno == EINVAL) break; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + debug("received daemon connection"); + + auto workerThread = std::thread([store, remote{std::move(remote)}]() { + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, + [&](Store & store) { store.createUser("nobody", 65535); }); + debug("terminated daemon connection"); + } catch (SysError &) { + ignoreException(); + } + }); + + daemonWorkerThreads.push_back(std::move(workerThread)); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationGoal::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) + throw SysError("shutting down daemon socket"); + + if (daemonThread.joinable()) + daemonThread.join(); + + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + + daemonSocket = -1; +} + + +void DerivationGoal::addDependency(const StorePath & path) +{ + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); + + #if __linux__ + + Path source = worker.store.Store::toRealPath(path); + Path target = chrootRootDir + worker.store.printStorePath(path); + debug("bind-mounting %s -> %s", target, source); + + if (pathExists(target)) + throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + + auto st = lstat(source); + + if (S_ISDIR(st.st_mode)) { + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + createDirs(target); + + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); + + } else + linkOrCopy(source, target); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + worker.store.printStorePath(path)); + #endif + + } +} + + +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError("cannot change ownership of '%1%'", path); +} + + +void setupSeccomp() +{ +#if __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + + +void DerivationGoal::runChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + try { /* child */ + + commonChildInit(builderOut); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + + bool setUser = true; + + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + netrcData = readFile(settings.netrcFile); + } catch (SysError &) { } + +#if __linux__ + if (useChroot) { + + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (privateNetwork) { + + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount '%1%'", chrootRootDir); + + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (derivationIsImpure(derivationType)) { + ss.push_back("/etc/resolv.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + ss.push_back("/etc/services"); + ss.push_back("/etc/hosts"); + if (pathExists("/var/run/nscd/socket")) + ss.push_back("/var/run/nscd/socket"); + } + + for (auto & i : ss) dirsInChroot.emplace(i, i); + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") + && !dirsInChroot.count("/dev/pts")) + { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError("cannot change directory to '%1%'", chrootRootDir); + + if (mkdir("real-root", 0) == -1) + throw SysError("cannot create real-root directory"); + + if (pivot_root(".", "real-root") == -1) + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); + + if (chroot(".") == -1) + throw SysError("cannot change root directory to '%1%'", chrootRootDir); + + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); + + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid()) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid()) == -1) + throw SysError("setuid failed"); + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv->platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set i686-linux personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto & i : env) + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + Strings args; + + const char *builder = "invalid"; + + if (drv->isBuiltin()) { + ; + } +#if __APPLE__ + else { + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + dirsInChroot[p] = p; + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + + if (derivationIsImpure(derivationType)) + sandboxProfile += "(import \"sandbox-network.sb\")\n"; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : dirsInChroot) { + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; + throw SysError("getting attributes of path '%s", path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = tmpDir + "/.sandbox.sb"; + + writeFile(sandboxFile, sandboxProfile); + + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); + } else { + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); + } + } +#else + else { + builder = drv->builder.c_str(); + args.push_back(std::string(baseNameOf(drv->builder))); + } +#endif + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, string("\2\n")); + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(*logger); + + BasicDerivation & drv2(*drv); + for (auto & e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); + + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); + else + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, e.what() + std::string("\n")); + _exit(1); + } + } + +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else + execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif + + throw SysError("executing '%1%'", drv->builder); + + } catch (Error & e) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + _exit(1); + } +} + + +void DerivationGoal::registerOutputs() +{ + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. + + We can only early return when the outputs are known a priori. For + floating content-addressed derivations this isn't the case. + */ + if (hook) { + bool allValid = true; + for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!outputPath || !worker.store.isValidPath(*outputPath)) + allValid = false; + else + finalOutputs.insert_or_assign(outputName, *outputPath); + } + if (allValid) return; + } + + std::map infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; + + std::exception_ptr delayedException; + + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); + for (auto & p : addedPaths) referenceablePaths.insert(p); + + /* FIXME `needsHashRewrite` should probably be removed and we get to the + real reason why we aren't using the chroot dir */ + auto toRealPathChroot = [&](const Path & p) -> Path { + return useChroot && !needsHashRewrite() + ? chrootRootDir + p + : worker.store.toRealPath(p); + }; + + /* Check whether the output paths were created, and make all + output paths read-only. Then get the references of each output (that we + might need to register), so we can topologically sort them. For the ones + that are most definitely already installed, we just store their final + name so we can also use it in rewrites. */ + StringSet outputsToSort; + struct AlreadyRegistered { StorePath path; }; + struct PerhapsNeedToRegister { StorePathSet refs; }; + std::map> outputReferencesIfUnregistered; + std::map outputStats; + for (auto & [outputName, _] : drv->outputs) { + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + + outputsToSort.insert(outputName); + + /* Updated wanted info to remove the outputs we definitely don't need to register */ + auto & initialInfo = initialOutputs.at(outputName); + + /* Don't register if already valid, and not checking */ + initialInfo.wanted = buildMode == bmCheck + || !(initialInfo.known && initialInfo.known->isValid()); + if (!initialInfo.wanted) { + outputReferencesIfUnregistered.insert_or_assign( + outputName, + AlreadyRegistered { .path = initialInfo.known->path }); + continue; + } + + struct stat st; + if (lstat(actualPath.c_str(), &st) == -1) { + if (errno == ENOENT) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); + throw SysError("getting attributes of path '%s'", actualPath); + } + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError( + "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", + actualPath, outputName); +#endif + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + auto references = worker.store.parseStorePathSet( + scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); + + outputReferencesIfUnregistered.insert_or_assign( + outputName, + PerhapsNeedToRegister { .refs = references }); + outputStats.insert_or_assign(outputName, std::move(st)); + } + + auto sortedOutputNames = topoSort(outputsToSort, + {[&](const std::string & name) { + return std::visit(overloaded { + /* Since we'll use the already installed versions of these, we + can treat them as leaves and ignore any references they + have. */ + [&](AlreadyRegistered _) { return StringSet {}; }, + [&](PerhapsNeedToRegister refs) { + StringSet referencedOutputs; + /* FIXME build inverted map up front so no quadratic waste here */ + for (auto & r : refs.refs) + for (auto & [o, p] : scratchOutputs) + if (r == p) + referencedOutputs.insert(o); + return referencedOutputs; + }, + }, outputReferencesIfUnregistered.at(name)); + }}, + {[&](const std::string & path, const std::string & parent) { + // TODO with more -vvvv also show the temporary paths for manual inspection. + return BuildError( + "cycle detected in build of '%s' in the references of output '%s' from output '%s'", + worker.store.printStorePath(drvPath), path, parent); + }}); + + std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + + for (auto & outputName : sortedOutputNames) { + auto output = drv->outputs.at(outputName); + auto & scratchPath = scratchOutputs.at(outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + + auto finish = [&](StorePath finalStorePath) { + /* Store the final path */ + finalOutputs.insert_or_assign(outputName, finalStorePath); + /* The rewrite rule will be used in downstream outputs that refer to + use. This is why the topological sort is essential to do first + before this for loop. */ + if (scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + }; + + std::optional referencesOpt = std::visit(overloaded { + [&](AlreadyRegistered skippedFinalPath) -> std::optional { + finish(skippedFinalPath.path); + return std::nullopt; + }, + [&](PerhapsNeedToRegister r) -> std::optional { + return r.refs; + }, + }, outputReferencesIfUnregistered.at(outputName)); + + if (!referencesOpt) + continue; + auto references = *referencesOpt; + + auto rewriteOutput = [&]() { + /* Apply hash rewriting if necessary. */ + if (!outputRewrites.empty()) { + warn("rewriting hashes in '%1%'; cross fingers", actualPath); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); + } + }; + + auto rewriteRefs = [&]() -> std::pair { + /* In the CA case, we need the rewritten refs to calculate the + final path, therefore we look for a *non-rewritten + self-reference, and use a bool rather try to solve the + computationally intractable fixed point. */ + std::pair res { + false, + {}, + }; + for (auto & r : references) { + auto name = r.name(); + auto origHash = std::string { r.hashPart() }; + if (r == scratchPath) + res.first = true; + else if (outputRewrites.count(origHash) == 0) + res.second.insert(r); + else { + std::string newRef = outputRewrites.at(origHash); + newRef += '-'; + newRef += name; + res.second.insert(StorePath { newRef }); + } + } + return res; + }; + + auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto & st = outputStats.at(outputName); + if (outputHash.method == FileIngestionMethod::Flat) { + /* The output path should be a regular file without execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + actualPath); + } + rewriteOutput(); + /* FIXME optimize and deduplicate with addToStore */ + std::string oldHashPart { scratchPath.hashPart() }; + HashModuloSink caSink { outputHash.hashType, oldHashPart }; + switch (outputHash.method) { + case FileIngestionMethod::Recursive: + dumpPath(actualPath, caSink); + break; + case FileIngestionMethod::Flat: + readFile(actualPath, caSink); + break; + } + auto got = caSink.finish().first; + auto refs = rewriteRefs(); + HashModuloSink narSink { htSHA256, oldHashPart }; + dumpPath(actualPath, narSink); + auto narHashAndSize = narSink.finish(); + ValidPathInfo newInfo0 { + worker.store.makeFixedOutputPath( + outputHash.method, + got, + outputPathName(drv->name, outputName), + refs.second, + refs.first), + narHashAndSize.first, + }; + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + if (scratchPath != newInfo0.path) { + // Also rewrite the output path + auto source = sinkToSource([&](Sink & nextSink) { + StringSink sink; + dumpPath(actualPath, sink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + rsink2(*sink.s); + rsink2.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + } + + assert(newInfo0.ca); + return newInfo0; + }; + + ValidPathInfo newInfo = std::visit(overloaded { + [&](DerivationOutputInputAddressed output) { + /* input-addressed case */ + auto requiredFinalPath = output.path; + /* Preemptively add rewrite rule for final hash, as that is + what the NAR hash will use rather than normalized-self references */ + if (scratchPath != requiredFinalPath) + outputRewrites.insert_or_assign( + std::string { scratchPath.hashPart() }, + std::string { requiredFinalPath.hashPart() }); + rewriteOutput(); + auto narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; + newInfo0.narSize = narHashAndSize.second; + auto refs = rewriteRefs(); + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + return newInfo0; + }, + [&](DerivationOutputCAFixed dof) { + auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + .method = dof.hash.method, + .hashType = dof.hash.hash.type, + }); + + /* Check wanted hash */ + Hash & wanted = dof.hash.hash; + assert(newInfo0.ca); + auto got = getContentAddressHash(*newInfo0.ca); + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + worker.store.printStorePath(drvPath), + wanted.to_string(SRI, true), + got.to_string(SRI, true))); + } + return newInfo0; + }, + [&](DerivationOutputCAFloating dof) { + return newInfoFromCA(dof); + }, + [&](DerivationOutputDeferred) { + // No derivation should reach that point without having been + // rewritten first + assert(false); + // Ugly, but the compiler insists on having this return a value + // of type `ValidPathInfo` despite the `assert(false)`, so + // let's provide it + return *(ValidPathInfo*)0; + }, + }, output.output); + + /* Calculate where we'll move the output files. In the checking case we + will leave leave them where they are, for now, rather than move to + their usual "final destination" */ + auto finalDestPath = worker.store.printStorePath(newInfo.path); + + /* Lock final output path, if not already locked. This happens with + floating CA derivations and hash-mismatching fixed-output + derivations. */ + PathLocks dynamicOutputLock; + auto optFixedPath = output.path(worker.store, drv->name, outputName); + if (!optFixedPath || + worker.store.printStorePath(*optFixedPath) != finalDestPath) + { + assert(newInfo.ca); + dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); + } + + /* Move files, if needed */ + if (worker.store.toRealPath(finalDestPath) != actualPath) { + if (buildMode == bmRepair) { + /* Path already exists, need to replace it */ + replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); + actualPath = worker.store.toRealPath(finalDestPath); + } else if (buildMode == bmCheck) { + /* Path already exists, and we want to compare, so we leave out + new path in place. */ + } else if (worker.store.isValidPath(newInfo.path)) { + /* Path already exists because CA path produced by something + else. No moving needed. */ + assert(newInfo.ca); + } else { + auto destPath = worker.store.toRealPath(finalDestPath); + movePath(actualPath, destPath); + actualPath = destPath; + } + } + + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); + auto & localStore = *localStoreP; + + if (buildMode == bmCheck) { + + if (!worker.store.isValidPath(newInfo.path)) continue; + ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { + auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); + deletePath(dst); + movePath(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); + + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); + } else + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); + } + + /* Since we verified the build, it's now ultimately trusted. */ + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (auto & i : inputPaths) { + auto j = references.find(i); + if (j == references.end()) + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); + else + debug("referenced input: '%1%'", worker.store.printStorePath(i)); + } + + if (curRound == nrRounds) { + localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); + } + + newInfo.deriver = drvPath; + newInfo.ultimate = true; + localStore.signPathInfo(newInfo); + + finish(newInfo.path); + + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration */ + if (newInfo.ca) + localStore.registerValidPaths({{newInfo.path, newInfo}}); + + infos.emplace(outputName, std::move(newInfo)); + } + + if (buildMode == bmCheck) return; + + /* Apply output checks. */ + checkOutputs(infos); + + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + result.isNonDeterministic = true; + Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; + bool prevExists = keepPreviousRound && pathExists(prev); + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) + : hintfmt("output '%s' of '%s' differs from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, worker.store.printStorePath(i->second.path), + worker.store.printStorePath(drvPath), tmpDir); + + if (settings.enforceDeterminism) + throw NotDeterministic(hint); + + printError(hint); + + curRound = nrRounds; // we know enough, bail out early + } + } + + /* If this is the first round of several, then move the output out of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { + for (auto & [_, outputStorePath] : finalOutputs) { + auto path = worker.store.printStorePath(outputStorePath); + Path prev = path + checkSuffix; + deletePath(prev); + Path dst = path + checkSuffix; + if (rename(path.c_str(), dst.c_str())) + throw SysError("renaming '%s' to '%s'", path, dst); + } + } + + if (curRound < nrRounds) { + prevInfos = std::move(infos); + return; + } + + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & [_, outputStorePath] : finalOutputs) { + Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; + deletePath(prev); + } + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); + auto & localStore = *localStoreP; + + ValidPathInfos infos2; + for (auto & [outputName, newInfo] : infos) { + infos2.insert_or_assign(newInfo.path, newInfo); + } + localStore.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); + + /* If we made it this far, we are sure the output matches the derivation + (since the delayedException would be a fixed output CA mismatch). That + means it's safe to link the derivation to the output hash. We must do + that for floating CA derivations, which otherwise couldn't be cached, + but it's fine to do in all cases. */ + + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto& [outputName, newInfo] : infos) + worker.store.registerDrvOutput(Realisation{ + .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, + .outPath = newInfo.path}); + } +} + + +void DerivationGoal::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) + { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(worker.store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) + { + if (!value) return; + + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : *value) { + if (worker.store.isStorePath(i)) + spec.insert(worker.store.parseStorePath(i)); + else if (finalOutputs.count(i)) + spec.insert(finalOutputs.at(i)); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); + } + + auto used = recursive + ? getClosure(info.path).first + : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += worker.store.printStorePath(i); + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + worker.store.printStorePath(info.path), badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get(); + + auto get = [&](const std::string & name) -> std::optional { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); + res.push_back(j->get()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + +Path DerivationGoal::openLogFile() +{ + logSize = 0; + + if (!settings.keepLog) return ""; + + auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); + + /* Create a log file. */ + Path logDir; + if (auto localStore = dynamic_cast(&worker.store)) + logDir = localStore->logDir; + else + logDir = settings.nixLogDir; + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); + createDirs(dir); + + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); + + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); + + logFileSink = std::make_shared(fdLogFile.get()); + + if (settings.compressLog) + logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; + + return logFileName; +} + + +void DerivationGoal::closeLogFile() +{ + auto logSink2 = std::dynamic_pointer_cast(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; + fdLogFile = -1; +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError("note: keeping build directory '%s'", tmpDir); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::handleChildOutput(int fd, const string & data) +{ + if ((hook && fd == hook->builderOut.readSide.get()) || + (!hook && fd == builderOut.readSide.get())) + { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + killChild(); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); + return; + } + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); + } + + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } +} + + +void DerivationGoal::handleEOF(int fd) +{ + if (!currentLogLine.empty()) flushLine(); + worker.wakeUp(shared_from_this()); +} + + +void DerivationGoal::flushLine() +{ + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; + + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); + + act->result(resBuildLogLine, currentLogLine); + } + + currentLogLine = ""; + currentLogLinePos = 0; +} + + +std::map> DerivationGoal::queryPartialDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + std::map> res; + for (auto & [name, output] : drv->outputs) + res.insert_or_assign(name, output.path(worker.store, drv->name, name)); + return res; + } else { + return worker.store.queryPartialDerivationOutputMap(drvPath); + } +} + +OutputPathMap DerivationGoal::queryDerivationOutputMap() +{ + if (!useDerivation || drv->type() != DerivationType::CAFloating) { + OutputPathMap res; + for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) + res.insert_or_assign(name, *output.second); + return res; + } else { + return worker.store.queryDerivationOutputMap(drvPath); + } +} + + +void DerivationGoal::checkPathValidity() +{ + bool checkHash = buildMode == bmRepair; + for (auto & i : queryPartialDerivationOutputMap()) { + InitialOutput & info = initialOutputs.at(i.first); + info.wanted = wantOutput(i.first, wantedOutputs); + if (i.second) { + auto outputPath = *i.second; + info.known = { + .path = outputPath, + .status = !worker.store.isValidPath(outputPath) + ? PathStatus::Absent + : !checkHash || worker.pathContentsGood(outputPath) + ? PathStatus::Valid + : PathStatus::Corrupt, + }; + } + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (auto real = worker.store.queryRealisation( + DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { + info.known = { + .path = real->outPath, + .status = PathStatus::Valid, + }; + } + } + } +} + + +StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + Hash(htSHA256), outputPathName(drv->name, outputName)); +} + + +StorePath DerivationGoal::makeFallbackPath(const StorePath & path) +{ + return worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + Hash(htSHA256), path.name()); +} + + +void DerivationGoal::done(BuildResult::Status status, std::optional ex) +{ + result.status = status; + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); + if (result.status == BuildResult::TimedOut) + worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) + worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) + worker.failedBuilds++; + } + + worker.updateProgress(); +} + + +} diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh new file mode 100644 index 000000000..6dc164922 --- /dev/null +++ b/src/libstore/build/local-derivation-goal.hh @@ -0,0 +1,373 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "lock.hh" +#include "local-store.hh" +#include "goal.hh" + +namespace nix { + +using std::map; + +struct HookInstance; + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + +/* Unless we are repairing, we don't both to test validity and just assume it, + so the choices are `Absent` or `Valid`. */ +enum struct PathStatus { + Corrupt, + Absent, + Valid, +}; + +struct InitialOutputStatus { + StorePath path; + PathStatus status; + /* Valid in the store, and additionally non-corrupt if we are repairing */ + bool isValid() const { + return status == PathStatus::Valid; + } + /* Merely present, allowed to be corrupt */ + bool isPresent() const { + return status == PathStatus::Corrupt + || status == PathStatus::Valid; + } +}; + +struct InitialOutput { + bool wanted; + Hash outputHash; + std::optional known; +}; + +struct DerivationGoal : public Goal +{ + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + + /* The path of the derivation. */ + StorePath drvPath; + + /* The path of the corresponding resolved derivation */ + std::optional resolvedDrv; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart = false; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + std::unique_ptr drv; + + std::unique_ptr parsedDrv; + + /* The remainder is state held during the build. */ + + /* Locks on (fixed) output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + StorePathSet inputPaths; + + std::map initialOutputs; + + /* User selected for running the builder. */ + std::unique_ptr buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + std::shared_ptr logFileSink, logSink; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* The most recent log lines. */ + std::list logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + + std::string currentHookLine; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* Pipe for synchronising updates to the builder namespaces. */ + Pipe userNamespaceSync; + + /* The mount namespace of the builder, used to add additional + paths to the sandbox as a result of recursive Nix calls. */ + AutoCloseFD sandboxMountNamespace; + + /* On Linux, whether we're doing the build in its own user + namespace. */ + bool usingUserNamespace = true; + + /* The build hook. */ + std::unique_ptr hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot = false; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + std::shared_ptr autoDelChroot; + + /* The sort of derivation we are building. */ + DerivationType derivationType; + + /* Whether to run the build in a private network namespace. */ + bool privateNetwork = false; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; + + typedef map Environment; + Environment env; + +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; +#endif + + /* Hash rewriting. */ + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /* The outputs paths used during the build. + + - Input-addressed derivations or fixed content-addressed outputs are + sometimes built when some of their outputs already exist, and can not + be hidden via sandboxing. We use temporary locations instead and + rewrite after the build. Otherwise the regular predetermined paths are + put here. + + - Floating content-addressed derivations do not know their final build + output paths until the outputs are hashed, so random locations are + used, and then renamed. The randomness helps guard against hidden + self-references. + */ + OutputPathMap scratchOutputs; + + /* The final output paths of the build. + + - For input-addressed derivations, always the precomputed paths + + - For content-addressed derivations, calcuated from whatever the hash + ends up being. (Note that fixed outputs derivations that produce the + "wrong" output still install that data under its true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + /* If we're repairing without a chroot, there may be outputs that + are valid but corrupt. So we redirect these outputs to + temporary paths. */ + StorePathSet redirectedBadOutputs; + + BuildResult result; + + /* The current round, if we're building multiple times. */ + size_t curRound = 1; + + size_t nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + std::map prevInfos; + + uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } + + const static Path homeDir; + + std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; + + std::unique_ptr act; + + /* Activity that denotes waiting for a lock. */ + std::unique_ptr actLock; + + std::map builderActivities; + + /* The remote machine on which we're building. */ + std::string machineName; + + /* The recursive Nix daemon socket. */ + AutoCloseFD daemonSocket; + + /* The daemon main thread. */ + std::thread daemonThread; + + /* The daemon worker threads. */ + std::vector daemonWorkerThreads; + + /* Paths that were added via recursive Nix calls. */ + StorePathSet addedPaths; + + /* Recursive Nix calls are only allowed to build or realize paths + in the original input closure or added via a recursive Nix call + (so e.g. you can't do 'nix-store -r /nix/store/' where + /nix/store/ is some arbitrary path in a binary cache). */ + bool isAllowed(const StorePath & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + + friend struct RestrictedStore; + + DerivationGoal(const StorePath & drvPath, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, + const StringSet & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + ~DerivationGoal(); + + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + + void timedOut(Error && ex) override; + + string key() override; + + void work() override; + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + + BuildResult getResult() { return result; } + + /* The states. */ + void getDerivation(); + void loadDerivation(); + void haveDerivation(); + void outputsSubstitutionTried(); + void gaveUpOnSubstitution(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void tryLocalBuild(); + void buildDone(); + + void resolvedFinished(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Setup tmp dir location. */ + void initTmpDir(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + void startDaemon(); + + void stopDaemon(); + + /* Add 'path' to the set of paths that may be referenced by the + outputs, and make it appear in the sandbox. */ + void addDependency(const StorePath & path); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path & path); + + /* Run the builder's process. */ + void runChild(); + + /* Check that the derivation outputs all exist and register them + as valid. */ + void registerOutputs(); + + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map & outputs); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + void flushLine(); + + /* Wrappers around the corresponding Store methods that first consult the + derivation. This is currently needed because when there is no drv file + there also is no DB entry. */ + std::map> queryPartialDerivationOutputMap(); + OutputPathMap queryDerivationOutputMap(); + + /* Return the set of (in)valid paths. */ + void checkPathValidity(); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + /* Create alternative path calculated from but distinct from the + input, so we can avoid overwriting outputs (or other store paths) + that already exist. */ + StorePath makeFallbackPath(const StorePath & path); + /* Make a path to another based on the output name along with the + derivation hash. */ + /* FIXME add option to randomize, so we can audit whether our + rewrites caught everything */ + StorePath makeFallbackPath(std::string_view outputName); + + void repairClosure(); + + void started(); + + void done( + BuildResult::Status status, + std::optional ex = {}); + + StorePathSet exportReferences(const StorePathSet & storePaths); +}; + +} From 68f4c728eca33f115f90e3f924c9081a4cd59896 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 15:20:33 +0000 Subject: [PATCH 159/725] Split {,local-}derivation-goal.{cc,hh} This separates the scheduling logic (including simple hook pathway) from the local-store needing code. This should be the final split for now. I'm reasonably happy with how it's turning out, even before I'm done moving code into `local-derivation-goal`. Benefits: 1. This will help "witness" that the hook case is indeed a lot simpler, and also compensate for the increased complexity that comes from content-addressed derivation outputs. 2. It also moves us ever so slightly towards a world where we could use off-the-shelf storage or sandboxing, since `local-derivation-goal` would be gutted in those cases, but `derivation-goal` should remain nearly the same. The new `#if 0` in the new files will be deleted in the following commit. I keep it here so if it turns out more stuff can be moved over, it's easy to do so in a way that preserves ordering --- and thus prevents conflicts. N.B. ```sh git diff HEAD^^ --color-moved --find-copies-harder --patience --stat ``` makes nicer output. --- src/libstore/build/derivation-goal.cc | 2741 +------------------ src/libstore/build/derivation-goal.hh | 186 +- src/libstore/build/entry-points.cc | 1 + src/libstore/build/local-derivation-goal.cc | 330 +-- src/libstore/build/local-derivation-goal.hh | 100 +- src/libstore/build/worker.cc | 14 +- src/libstore/local-store.hh | 2 +- 7 files changed, 329 insertions(+), 3045 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 924c69fb7..c29237f5c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -9,10 +9,10 @@ #include "archive.hh" #include "json.hh" #include "compression.hh" -#include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" +#include "local-store.hh" // TODO remove, along with remaining downcasts #include #include @@ -62,40 +62,6 @@ namespace nix { -void handleDiffHook( - uid_t uid, uid_t gid, - const Path & tryA, const Path & tryB, - const Path & drvPath, const Path & tmpDir) -{ - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { - try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - - auto diffRes = runProgram(diffHookOptions); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, - "diff-hook program '%1%' %2%", - diffHook, - statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - // FIXME: wrap errors. - ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); - logError(ei); - } - } -} - -const Path DerivationGoal::homeDir = "/homeless-shelter"; - DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -144,9 +110,6 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { closeLogFile(); } catch (...) { ignoreException(); } } @@ -161,38 +124,8 @@ string DerivationGoal::key() } -inline bool DerivationGoal::needsHashRewrite() -{ -#if __linux__ - return !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - return true; -#endif -} - - void DerivationGoal::killChild() { - if (pid != -1) { - worker.childTerminated(this); - - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); - - assert(pid == -1); - } - hook.reset(); } @@ -697,64 +630,10 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - /* Make sure that we are allowed to start a build. */ - if (!dynamic_cast(&worker.store)) { - throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - } - unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { - worker.waitForBuildSlot(shared_from_this()); - outputLocks.unlock(); - return; - } - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return; - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - - actLock.reset(); - - try { - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - outputLocks.unlock(); - buildUser.reset(); - worker.permanentFailure = true; - done(BuildResult::InputRejected, e); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - - started(); + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); } @@ -811,25 +690,63 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) } -MakeError(NotDeterministic, BuildError); +int DerivationGoal::getChildStatus() +{ + return hook->pid.kill(); +} + + +void DerivationGoal::closeReadPipes() +{ + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; +} + + +void DerivationGoal::cleanupHookFinally() +{ +} + + +void DerivationGoal::cleanupPreChildKill() +{ +} + + +void DerivationGoal::cleanupPostChildKill() +{ +} + + +bool DerivationGoal::cleanupDecideWhetherDiskFull() +{ + return false; +} + + +void DerivationGoal::cleanupPostOutputsRegisteredModeCheck() +{ +} + + +void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() +{ +} void DerivationGoal::buildDone() { trace("build done"); - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); + Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - sandboxMountNamespace = -1; + cleanupPreChildKill(); /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); + int status = getChildStatus(); debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); @@ -840,24 +757,12 @@ void DerivationGoal::buildDone() worker.childTerminated(this); /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; + closeReadPipes(); /* Close the log file. */ closeLogFile(); - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); + cleanupPostChildKill(); bool diskFull = false; @@ -866,36 +771,7 @@ void DerivationGoal::buildDone() /* Check the exit status. */ if (!statusOk(status)) { - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - if (auto localStore = dynamic_cast(&worker.store)) { - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } + diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", yellowtxt(worker.store.printStorePath(drvPath)), @@ -975,19 +851,12 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeCheck(); done(BuildResult::Built); return; } - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { @@ -1171,13 +1040,6 @@ HookReply DerivationGoal::tryBuildHook() } -int childEntry(void * arg) -{ - ((DerivationGoal *) arg)->runChild(); - return 1; -} - - StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { StorePathSet paths; @@ -1213,1769 +1075,6 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) return paths; } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - -void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} - - -void DerivationGoal::startBuilder() -{ - /* Right platform? */ - if (!parsedDrv->canBuildLocally(worker.store)) - throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", - drv->platform, - concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - worker.store.printStorePath(drvPath), - settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); - - if (drv->isBuiltin()) - preloadNSS(); - -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; - } - - if (auto localStoreP = dynamic_cast(&worker.store)) { - auto & localStore = *localStoreP; - if (localStore.storeDir != localStore.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - } - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); - - chownToBuilder(tmpDir); - - for (auto & [outputName, status] : initialOutputs) { - /* Set scratch path we'll actually use during the build. - - If we're not doing a chroot build, but we have some valid - output paths. Since we can't just overwrite or delete - them, we have to do hash rewriting: i.e. in the - environment/arguments passed to the build, we replace the - hashes of the valid outputs with unique dummy strings; - after the build, we discard the redirected outputs - corresponding to the valid outputs, and rewrite the - contents of the new outputs to replace the dummy strings - with the actual hashes. */ - auto scratchPath = - !status.known - ? makeFallbackPath(outputName) - : !needsHashRewrite() - /* Can always use original path in sandbox */ - ? status.known->path - : !status.known->isPresent() - /* If path doesn't yet exist can just use it */ - ? status.known->path - : buildMode != bmRepair && !status.known->isValid() - /* If we aren't repairing we'll delete a corrupted path, so we - can use original path */ - ? status.known->path - : /* If we are repairing or the path is totally valid, we'll need - to use a temporary path */ - makeFallbackPath(status.known->path); - scratchOutputs.insert_or_assign(outputName, scratchPath); - - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - - /* Substitute output placeholders with the scratch output paths. - We'll use during the build. */ - inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); - - /* Additional tasks if we know the final path a priori. */ - if (!status.known) continue; - auto fixedFinalPath = status.known->path; - - /* Additional tasks if the final and scratch are both known and - differ. */ - if (fixedFinalPath == scratchPath) continue; - - /* Ensure scratch path is ours to use. */ - deletePath(worker.store.printStorePath(scratchPath)); - - /* Rewrite and unrewrite paths */ - { - std::string h1 { fixedFinalPath.hashPart() }; - std::string h2 { scratchPath.hashPart() }; - inputRewrites[h1] = h2; - } - - redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath)); - } - - /* Construct the environment passed to the builder. */ - initEnv(); - - writeStructuredAttrs(); - - /* Handle exportReferencesGraph(), if set. */ - if (!parsedDrv->getStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - - /* Write closure info to . */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); - } - } - - if (useChroot) { - - /* Allow a user-configurable set of directories from the - host file system. */ - dirsInChroot.clear(); - - for (auto i : settings.sandboxPaths.get()) { - if (i.empty()) continue; - bool optional = false; - if (i[i.size() - 1] == '?') { - optional = true; - i.pop_back(); - } - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = {i, optional}; - else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; - } - dirsInChroot[tmpDirInSandbox] = tmpDir; - - /* Add the closure of store paths to the chroot. */ - StorePathSet closure; - for (auto & i : dirsInChroot) - try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); - } catch (InvalidPath & e) { - } catch (Error & e) { - e.addTrace({}, "while processing 'sandbox-paths'"); - throw; - } - for (auto & i : closure) { - auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); - } - - PathSet allowedPaths = settings.allowedImpureHostPrefixes; - - /* This works like the above, except on a per-derivation level */ - auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); - - for (auto & i : impurePaths) { - bool found = false; - /* Note: we're not resolving symlinks here to prevent - giving a non-root user info about inaccessible - files. */ - Path canonI = canonPath(i); - /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ - for (auto & a : allowedPaths) { - Path canonA = canonPath(a); - if (canonI == canonA || isInDir(canonI, canonA)) { - found = true; - break; - } - } - if (!found) - throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", - worker.store.printStorePath(drvPath), i); - - dirsInChroot[i] = i; - } - -#if __linux__ - /* Create a temporary directory in which we set up the chroot - environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); - - /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); - - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - - if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); - - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); - - /* Create a writable /tmp in the chroot. Many builders need - this. (Of course they should really respect $TMPDIR - instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; - createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); - - /* Create a /etc/passwd with entries for the build user and the - nobody account. The latter is kind of a hack to support - Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile(chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", sandboxGid())); - - /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); - - /* Make the closure of the inputs available in the chroot, - rather than the whole Nix store. This prevents any access - to undeclared dependencies. Directories are bind-mounted, - while other inputs are hard-linked (since only directories - can be bind-mounted). !!! As an extra security - precaution, make the fake Nix store only writable by the - build user. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); - - if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); - - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); - } - - /* If we're repairing, checking or rebuilding part of a - multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot - (typically the dependencies of /bin/sh). Throw them - out. */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - /* If the name isn't known a priori (i.e. floating - content-addressed derivation), the temporary location we use - should be fresh. Freshness means it is impossible that the path - is already in the sandbox, so we don't need to worry about - removing it. */ - if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); - } - -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ -#else - throw Error("sandboxing builds is not supported on this platform"); -#endif - } - - if (needsHashRewrite() && pathExists(homeDir)) - throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); - - if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); - auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : - Strings({ worker.store.printStorePath(drvPath) }); - enum BuildHookState { - stBegin, - stExtraChrootDirs - }; - auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, args); - auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; - lastPos = nlPos + 1; - if (state == stBegin) { - if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { - state = stExtraChrootDirs; - } else { - throw Error("unknown pre-build hook command '%1%'", line); - } - } else if (state == stExtraChrootDirs) { - if (line == "") { - state = stBegin; - } else { - auto p = line.find('='); - if (p == string::npos) - dirsInChroot[line] = line; - else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); - } - } - } - } - - /* Fire up a Nix daemon to process recursive Nix calls from the - builder. */ - if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) - startDaemon(); - - /* Run the builder. */ - printMsg(lvlChatty, "executing builder '%1%'", drv->builder); - - /* Create the log file. */ - Path logFile = openLogFile(); - - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) - throw SysError("opening pseudoterminal master"); - - std::string slaveName(ptsname(builderOut.readSide.get())); - - if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); - - if (chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); - } -#if __APPLE__ - else { - if (grantpt(builderOut.readSide.get())) - throw SysError("granting access to pseudoterminal slave"); - } -#endif - - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) - throw SysError("unlocking pseudoterminal"); - - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - result.startTime = time(0); - - /* Fork a child to build the package. */ - ProcessOptions options; - -#if __linux__ - if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to those - on the inside, but processes inside the chroot are - visible from the outside (though with different PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and its - children, and will disappear automatically when we're - done. - - - The private network namespace ensures that the builder - cannot talk to the outside world (or vice versa). It - only has a private loopback interface. (Fixed-output - derivations are not run in a private network namespace - to allow functions like fetchurl to work.) - - - The IPC namespace prevents the builder from communicating - with outside processes using SysV IPC mechanisms (shared - memory, message queues, semaphores). It also ensures - that all IPC objects are destroyed when the builder - exits. - - - The UTS namespace ensures that builders see a hostname of - localhost rather than the actual hostname. - - We use a helper process to do the clone() to work around - clone() being broken in multi-threaded programs due to - at-fork handlers not being run. Note that we use - CLONE_PARENT to ensure that the real builder is parented to - us. - */ - - if (!(derivationIsImpure(derivationType))) - privateNetwork = true; - - userNamespaceSync.create(); - - options.allowVfork = false; - - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; - - Pid helper = startProcess([&]() { - - /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - flags |= CLONE_NEWNET; - if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); - _exit(0); - }, options); - - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) - throw Error("unable to start build process"); - - userNamespaceSync.readSide = -1; - - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { - userNamespaceSync.writeSide = -1; - }); - - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int(ss[1]).value(); - - if (usingUserNamespace) { - /* Set the UID/GID mapping of the builder's user namespace - such that the sandbox user maps to the build user, or to - the calling user (if build users are disabled). */ - uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); - uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - - writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d 1", sandboxUid(), hostUid)); - - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); - - writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d 1", sandboxGid(), hostGid)); - } else { - debug("note: not using a user namespace"); - if (!buildUser) - throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); - } - - /* Now that we now the sandbox uid, we can write - /etc/passwd. */ - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - - /* Save the mount namespace of the child. We have to do this - *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); - if (sandboxMountNamespace.get() == -1) - throw SysError("getting sandbox mount namespace"); - - /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); - - } else -#endif - { - fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); - pid = startProcess([&]() { - runChild(); - }, options); - } - - /* parent */ - pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); - - /* Check if setting up the build environment failed. */ - std::vector msgs; - while (true) { - string msg = [&]() { - try { - return readLine(builderOut.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", - concatStringsSep("|", msgs)); - throw e; - } - }(); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); - auto ex = readError(source); - ex.addTrace({}, "while setting up the build environment"); - throw ex; - } - debug("sandbox setup: " + msg); - msgs.push_back(std::move(msg)); - } -} - - -void DerivationGoal::initTmpDir() { - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif - - /* In non-structured mode, add all bindings specified in the - derivation via the environment, except those listed in the - passAsFile attribute. Those are passed as file names pointing - to temporary files containing the contents. Note that - passAsFile is ignored in structure mode because it's not - needed (attributes are not passed through the environment, so - there is no size constraint). */ - if (!parsedDrv->getStructuredAttrs()) { - - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); - Path p = tmpDir + "/" + fn; - writeFile(p, rewriteStrings(i.second, inputRewrites)); - chownToBuilder(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; -} - - -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - - initTmpDir(); - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); - } - - /* Currently structured log messages piggyback on stderr, but we - may change that in the future. So tell the builder which file - descriptor to use for that. */ - env["NIX_LOG_FD"] = "2"; - - /* Trigger colored output in various tools. */ - env["TERM"] = "xterm-256color"; -} - - -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - -void DerivationGoal::writeStructuredAttrs() -{ - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get(); - if (std::ceil(f) == f) - return std::to_string(value.get()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); -} - -struct RestrictedStoreConfig : virtual LocalFSStoreConfig -{ - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; - -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore -{ - ref next; - - DerivationGoal & goal; - - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) - { } - - Path getRealStoreDir() override - { return next->realStoreDir; } - - std::string getUri() override - { return next->getUri(); } - - StorePathSet queryAllValidPaths() override - { - StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); - return paths; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - if (goal.isAllowed(path)) { - try { - /* Censor impure information. */ - auto info = std::make_shared(*next->queryPathInfo(path)); - info->deriver.reset(); - info->registrationTime = 0; - info->ultimate = false; - info->sigs.clear(); - callback(info); - } catch (InvalidPath &) { - callback(nullptr); - } - } else - callback(nullptr); - }; - - void queryReferrers(const StorePath & path, StorePathSet & referrers) override - { } - - std::map> queryPartialDerivationOutputMap(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { throw Error("queryPathFromHashPart"); } - - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); } - - void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override - { - next->addToStore(info, narSource, repair, checkSigs); - goal.addDependency(info.path); - } - - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); - } - - void ensurePath(const StorePath & path) override - { - if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); - /* Nothing to be done; 'path' must already be valid. */ - } - - void registerDrvOutput(const Realisation & info) override - // XXX: This should probably be allowed as a no-op if the realisation - // corresponds to an allowed derivation - { throw Error("registerDrvOutput"); } - - std::optional queryRealisation(const DrvOutput & id) override - // XXX: This should probably be allowed if the realisation corresponds to - // an allowed derivation - { throw Error("queryRealisation"); } - - void buildPaths(const std::vector & paths, BuildMode buildMode) override - { - if (buildMode != bmNormal) throw Error("unsupported build mode"); - - StorePathSet newPaths; - - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); - } - - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); - } - - StorePathSet closure; - next->computeFSClosure(newPaths, closure); - for (auto & path : closure) - goal.addDependency(path); - } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override - { unsupported("buildDerivation"); } - - void addTempRoot(const StorePath & path) override - { } - - void addIndirectRoot(const Path & path) override - { } - - Roots findRoots(bool censor) override - { return Roots(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { } - - void addSignatures(const StorePath & storePath, const StringSet & sigs) override - { unsupported("addSignatures"); } - - void queryMissing(const std::vector & targets, - StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - uint64_t & downloadSize, uint64_t & narSize) override - { - /* This is slightly impure since it leaks information to the - client about what paths will be built/substituted or are - already present. Probably not a big deal. */ - - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); - else - unknown.insert(path.path); - } - - next->queryMissing(allowed, willBuild, willSubstitute, - unknown, downloadSize, narSize); - } -}; - - -void DerivationGoal::startDaemon() -{ - settings.requireExperimentalFeature("recursive-nix"); - - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto localStore = dynamic_cast(&worker.store)) - params["root"] = localStore->rootDir; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, - ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); - - addedPaths.clear(); - - auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; - - daemonSocket = createUnixDomainSocket(socketPath, 0600); - - chownToBuilder(socketPath); - - daemonThread = std::thread([this, store]() { - - while (true) { - - /* Accept a connection. */ - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - if (!remote) { - if (errno == EINTR) continue; - if (errno == EINVAL) break; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - debug("received daemon connection"); - - auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); - debug("terminated daemon connection"); - } catch (SysError &) { - ignoreException(); - } - }); - - daemonWorkerThreads.push_back(std::move(workerThread)); - } - - debug("daemon shutting down"); - }); -} - - -void DerivationGoal::stopDaemon() -{ - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); - - if (daemonThread.joinable()) - daemonThread.join(); - - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) - thread.join(); - daemonWorkerThreads.clear(); - - daemonSocket = -1; -} - - -void DerivationGoal::addDependency(const StorePath & path) -{ - if (isAllowed(path)) return; - - addedPaths.insert(path); - - /* If we're doing a sandbox build, then we have to make the path - appear in the sandbox. */ - if (useChroot) { - - debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); - - #if __linux__ - - Path source = worker.store.Store::toRealPath(path); - Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - - if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); - - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); - - #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", - worker.store.printStorePath(path)); - #endif - - } -} - - -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); -} - - -void setupSeccomp() -{ -#if __linux__ - if (!settings.filterSyscalls) return; -#if HAVE_SECCOMP - scmp_filter_ctx ctx; - - if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) - throw SysError("unable to initialize seccomp mode 2"); - - Finally cleanup([&]() { - seccomp_release(ctx); - }); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) - throw SysError("unable to add 32-bit seccomp architecture"); - - if (nativeSystem == "x86_64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) - throw SysError("unable to add X32 seccomp architecture"); - - if (nativeSystem == "aarch64-linux" && - seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); - - /* Prevent builders from creating setuid/setgid binaries. */ - for (int perm : { S_ISUID, S_ISGID }) { - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, - SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, - SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) - throw SysError("unable to add seccomp rule"); - } - - /* Prevent builders from creating EAs or ACLs. Not all filesystems - support these, and they're not allowed in the Nix store because - they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || - seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) - throw SysError("unable to add seccomp rule"); - - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) - throw SysError("unable to set 'no new privileges' seccomp attribute"); - - if (seccomp_load(ctx) != 0) - throw SysError("unable to load seccomp BPF program"); -#else - throw Error( - "seccomp is not supported on this platform; " - "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); -#endif -#endif -} - - -void DerivationGoal::runChild() -{ - /* Warning: in the child we should absolutely not make any SQLite - calls! */ - - try { /* child */ - - commonChildInit(builderOut); - - try { - setupSeccomp(); - } catch (...) { - if (buildUser) throw; - } - - bool setUser = true; - - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ - std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } - -#if __linux__ - if (useChroot) { - - userNamespaceSync.writeSide = -1; - - if (drainFD(userNamespaceSync.readSide.get()) != "1") - throw Error("user namespace initialisation failed"); - - userNamespaceSync.readSide = -1; - - if (privateNetwork) { - - /* Initialise the loopback interface. */ - AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); - if (!fd) throw SysError("cannot open IP socket"); - - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); - ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; - if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) - throw SysError("cannot set loopback interface flags"); - } - - /* Set the hostname etc. to fixed values. */ - char hostname[] = "localhost"; - if (sethostname(hostname, sizeof(hostname)) == -1) - throw SysError("cannot set host name"); - char domainname[] = "(none)"; // kernel default - if (setdomainname(domainname, sizeof(domainname)) == -1) - throw SysError("cannot set domain name"); - - /* Make all filesystems private. This is necessary - because subtrees may have been mounted as "shared" - (MS_SHARED). (Systemd does this, for instance.) Even - though we have a private mount namespace, mounting - filesystems on top of a shared subtree still propagates - outside of the namespace. Making a subtree private is - local to the namespace, though, so setting MS_PRIVATE - does not affect the outside world. */ - if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) - throw SysError("unable to make '/' private"); - - /* Bind-mount chroot directory to itself, to treat it as a - different filesystem from /, as needed for pivot_root. */ - if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); - - /* Bind-mount the sandbox's Nix store onto itself so that - we can mark it as a "shared" subtree, allowing bind - mounts made in *this* mount namespace to be propagated - into the child namespace created by the - unshare(CLONE_NEWNS) call below. - - Marking chrootRootDir as MS_SHARED causes pivot_root() - to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + worker.store.storeDir; - - if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); - - if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); - - /* Set up a nearly empty /dev, unless the user asked to - bind-mount the host /dev. */ - Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); - ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); - ss.push_back("/dev/null"); - ss.push_back("/dev/random"); - ss.push_back("/dev/tty"); - ss.push_back("/dev/urandom"); - ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); - } - - /* Fixed-output derivations typically need to access the - network, so give them access to /etc/resolv.conf and so - on. */ - if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - - // Only use nss functions to resolve hosts and - // services. Don’t use it for anything else that may - // be configured for this system. This limits the - // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); - } - - for (auto & i : ss) dirsInChroot.emplace(i, i); - - /* Bind-mount all the directories from the "host" - filesystem that we want in the chroot - environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { - if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); - } - - /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) - throw SysError("mounting /proc"); - - /* Mount a new tmpfs on /dev/shm to ensure that whatever - the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) - throw SysError("mounting /dev/shm"); - - /* Mount a new devpts on /dev/pts. Note that this - requires the kernel to be compiled with - CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case - if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && - !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) - { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) - { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); - } else { - if (errno != EINVAL) - throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); - } - } - - /* Unshare this mount namespace. This is necessary because - pivot_root() below changes the root of the mount - namespace. This means that the call to setns() in - addDependency() would hide the host's filesystem, - making it impossible to bind-mount paths from the host - Nix store into the sandbox. Therefore, we save the - pre-pivot_root namespace in - sandboxMountNamespace. Since we made /nix/store a - shared subtree above, this allows addDependency() to - make paths appear in the sandbox. */ - if (unshare(CLONE_NEWNS) == -1) - throw SysError("unsharing mount namespace"); - - /* Do the chroot(). */ - if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); - - if (mkdir("real-root", 0) == -1) - throw SysError("cannot create real-root directory"); - - if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); - - if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); - - if (umount2("real-root", MNT_DETACH) == -1) - throw SysError("cannot unmount real root filesystem"); - - if (rmdir("real-root") == -1) - throw SysError("cannot remove real-root directory"); - - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); - - setUser = false; - } -#endif - - if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); - - /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif - - /* Disable core dumps by default. */ - struct rlimit limit = { 0, RLIM_INFINITY }; - setrlimit(RLIMIT_CORE, &limit); - - // FIXME: set other limits to deterministic values? - - /* Fill in the environment. */ - Strings envStrs; - for (auto & i : env) - envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); - - /* If we are running in `build-users' mode, then switch to the - user we allocated above. Make sure that we drop all root - privileges. Note that above we have closed all file - descriptors except std*, so that's safe. Also note that - setuid() when run as root sets the real, effective and - saved UIDs. */ - if (setUser && buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || - getgid() != buildUser->getGID() || - getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || - getuid() != buildUser->getUID() || - geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); - } - - /* Fill in the arguments. */ - Strings args; - - const char *builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } -#if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); - } - - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; - } - - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } - - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - - if (derivationIsImpure(derivationType)) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) - continue; - throw SysError("getting attributes of path '%s", path); - } - if (S_ISDIR(st.st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); - - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } - } -#else - else { - builder = drv->builder.c_str(); - args.push_back(std::string(baseNameOf(drv->builder))); - } -#endif - - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - - /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); - - /* Execute the program. This should not return. */ - if (drv->isBuiltin()) { - try { - logger = makeJSONLogger(*logger); - - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); - - if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); - else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); - else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); - else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); - _exit(0); - } catch (std::exception & e) { - writeFull(STDERR_FILENO, e.what() + std::string("\n")); - _exit(1); - } - } - -#if __APPLE__ - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) - throw SysError("failed to initialize builder"); - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) - throw SysError("failed to initialize builder"); - - if (drv->platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } - - posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#else - execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#endif - - throw SysError("executing '%1%'", drv->builder); - - } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - _exit(1); - } -} - void DerivationGoal::registerOutputs() { @@ -2986,698 +1085,23 @@ void DerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - if (hook) { - bool allValid = true; - for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!outputPath || !worker.store.isValidPath(*outputPath)) - allValid = false; - else - finalOutputs.insert_or_assign(outputName, *outputPath); - } - if (allValid) return; - } - - std::map infos; - - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - - Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; - - std::exception_ptr delayedException; - - /* The paths that can be referenced are the input closures, the - output paths, and any paths that have been built via recursive - Nix calls. */ - StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) referenceablePaths.insert(p); - - /* FIXME `needsHashRewrite` should probably be removed and we get to the - real reason why we aren't using the chroot dir */ - auto toRealPathChroot = [&](const Path & p) -> Path { - return useChroot && !needsHashRewrite() - ? chrootRootDir + p - : worker.store.toRealPath(p); - }; - - /* Check whether the output paths were created, and make all - output paths read-only. Then get the references of each output (that we - might need to register), so we can topologically sort them. For the ones - that are most definitely already installed, we just store their final - name so we can also use it in rewrites. */ - StringSet outputsToSort; - struct AlreadyRegistered { StorePath path; }; - struct PerhapsNeedToRegister { StorePathSet refs; }; - std::map> outputReferencesIfUnregistered; - std::map outputStats; - for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); - - outputsToSort.insert(outputName); - - /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); - - /* Don't register if already valid, and not checking */ - initialInfo.wanted = buildMode == bmCheck - || !(initialInfo.known && initialInfo.known->isValid()); - if (!initialInfo.wanted) { - outputReferencesIfUnregistered.insert_or_assign( - outputName, - AlreadyRegistered { .path = initialInfo.known->path }); + for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (!wantOutput(outputName, wantedOutputs)) continue; - } - - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } - -#ifndef __CYGWIN__ - /* Check that the output is not group or world writable, as - that means that someone else can have interfered with the - build. Also, the output should be owned by the build - user. */ - if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser && st.st_uid != buildUser->getUID())) + if (!optOutputPath) throw BuildError( - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, outputName); -#endif + "output '%s' from derivation '%s' does not have a known output path", + outputName, worker.store.printStorePath(drvPath)); + auto & outputPath = *optOutputPath; + if (!worker.store.isValidPath(outputPath)) + throw BuildError( + "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid", + outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath)); - /* Canonicalise first. This ensures that the path we're - rewriting doesn't contain a hard link to /etc/shadow or - something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); - - outputReferencesIfUnregistered.insert_or_assign( - outputName, - PerhapsNeedToRegister { .refs = references }); - outputStats.insert_or_assign(outputName, std::move(st)); - } - - auto sortedOutputNames = topoSort(outputsToSort, - {[&](const std::string & name) { - return std::visit(overloaded { - /* Since we'll use the already installed versions of these, we - can treat them as leaves and ignore any references they - have. */ - [&](AlreadyRegistered _) { return StringSet {}; }, - [&](PerhapsNeedToRegister refs) { - StringSet referencedOutputs; - /* FIXME build inverted map up front so no quadratic waste here */ - for (auto & r : refs.refs) - for (auto & [o, p] : scratchOutputs) - if (r == p) - referencedOutputs.insert(o); - return referencedOutputs; - }, - }, outputReferencesIfUnregistered.at(name)); - }}, - {[&](const std::string & path, const std::string & parent) { - // TODO with more -vvvv also show the temporary paths for manual inspection. - return BuildError( - "cycle detected in build of '%s' in the references of output '%s' from output '%s'", - worker.store.printStorePath(drvPath), path, parent); - }}); - - std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); - - for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); - - auto finish = [&](StorePath finalStorePath) { - /* Store the final path */ - finalOutputs.insert_or_assign(outputName, finalStorePath); - /* The rewrite rule will be used in downstream outputs that refer to - use. This is why the topological sort is essential to do first - before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; - }; - - std::optional referencesOpt = std::visit(overloaded { - [&](AlreadyRegistered skippedFinalPath) -> std::optional { - finish(skippedFinalPath.path); - return std::nullopt; - }, - [&](PerhapsNeedToRegister r) -> std::optional { - return r.refs; - }, - }, outputReferencesIfUnregistered.at(outputName)); - - if (!referencesOpt) - continue; - auto references = *referencesOpt; - - auto rewriteOutput = [&]() { - /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { - warn("rewriting hashes in '%1%'; cross fingers", actualPath); - - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); - } - }; - - auto rewriteRefs = [&]() -> std::pair { - /* In the CA case, we need the rewritten refs to calculate the - final path, therefore we look for a *non-rewritten - self-reference, and use a bool rather try to solve the - computationally intractable fixed point. */ - std::pair res { - false, - {}, - }; - for (auto & r : references) { - auto name = r.name(); - auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) - res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); - newRef += '-'; - newRef += name; - res.second.insert(StorePath { newRef }); - } - } - return res; - }; - - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); - if (outputHash.method == FileIngestionMethod::Flat) { - /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw BuildError( - "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", - actualPath); - } - rewriteOutput(); - /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2(*sink.s); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); - } - - assert(newInfo0.ca); - return newInfo0; - }; - - ValidPathInfo newInfo = std::visit(overloaded { - [&](DerivationOutputInputAddressed output) { - /* input-addressed case */ - auto requiredFinalPath = output.path; - /* Preemptively add rewrite rule for final hash, as that is - what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) - outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, - std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; - newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - return newInfo0; - }, - [&](DerivationOutputCAFixed dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, - }); - - /* Check wanted hash */ - Hash & wanted = dof.hash.hash; - assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - worker.hashMismatch = true; - delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); - } - return newInfo0; - }, - [&](DerivationOutputCAFloating dof) { - return newInfoFromCA(dof); - }, - [&](DerivationOutputDeferred) { - // No derivation should reach that point without having been - // rewritten first - assert(false); - // Ugly, but the compiler insists on having this return a value - // of type `ValidPathInfo` despite the `assert(false)`, so - // let's provide it - return *(ValidPathInfo*)0; - }, - }, output.output); - - /* Calculate where we'll move the output files. In the checking case we - will leave leave them where they are, for now, rather than move to - their usual "final destination" */ - auto finalDestPath = worker.store.printStorePath(newInfo.path); - - /* Lock final output path, if not already locked. This happens with - floating CA derivations and hash-mismatching fixed-output - derivations. */ - PathLocks dynamicOutputLock; - auto optFixedPath = output.path(worker.store, drv->name, outputName); - if (!optFixedPath || - worker.store.printStorePath(*optFixedPath) != finalDestPath) - { - assert(newInfo.ca); - dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)}); - } - - /* Move files, if needed */ - if (worker.store.toRealPath(finalDestPath) != actualPath) { - if (buildMode == bmRepair) { - /* Path already exists, need to replace it */ - replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath); - actualPath = worker.store.toRealPath(finalDestPath); - } else if (buildMode == bmCheck) { - /* Path already exists, and we want to compare, so we leave out - new path in place. */ - } else if (worker.store.isValidPath(newInfo.path)) { - /* Path already exists because CA path produced by something - else. No moving needed. */ - assert(newInfo.ca); - } else { - auto destPath = worker.store.toRealPath(finalDestPath); - movePath(actualPath, destPath); - actualPath = destPath; - } - } - - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; - - if (buildMode == bmCheck) { - - if (!worker.store.isValidPath(newInfo.path)) continue; - ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; - if (settings.runDiffHook || settings.keepFailed) { - auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir); - - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst); - } else - throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath)); - } - - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - localStore.signPathInfo(oldInfo); - localStore.registerValidPaths({{oldInfo.path, oldInfo}}); - } - - continue; - } - - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else - debug("referenced input: '%1%'", worker.store.printStorePath(i)); - } - - if (curRound == nrRounds) { - localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } - - newInfo.deriver = drvPath; - newInfo.ultimate = true; - localStore.signPathInfo(newInfo); - - finish(newInfo.path); - - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - localStore.registerValidPaths({{newInfo.path, newInfo}}); - - infos.emplace(outputName, std::move(newInfo)); - } - - if (buildMode == bmCheck) return; - - /* Apply output checks. */ - checkOutputs(infos); - - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - result.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. If there are cycles in the - outputs, this will fail. */ - { - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; - - ValidPathInfos infos2; - for (auto & [outputName, newInfo] : infos) { - infos2.insert_or_assign(newInfo.path, newInfo); - } - localStore.registerValidPaths(infos2); - } - - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ - - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) - worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, - .outPath = newInfo.path}); + finalOutputs.insert_or_assign(outputName, outputPath); } } - -void DerivationGoal::checkOutputs(const std::map & outputs) -{ - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - struct Checks - { - bool ignoreSelfRefs = false; - std::optional maxSize, maxClosureSize; - std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; - }; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) - { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; - - auto i = outputsByPath.find(worker.store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = worker.store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const Checks & checks) - { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); - } - - auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) - { - if (!value) return; - - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : *value) { - if (worker.store.isStorePath(i)) - spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); - else throw BuildError("derivation contains an illegal reference specifier '%s'", i); - } - - auto used = recursive - ? getClosure(info.path).first - : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += worker.store.printStorePath(i); - } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", - worker.store.printStorePath(info.path), badPathsStr); - } - }; - - checkRefs(checks.allowedReferences, true, false); - checkRefs(checks.allowedRequisites, true, true); - checkRefs(checks.disallowedReferences, false, false); - checkRefs(checks.disallowedRequisites, false, true); - }; - - if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { - Checks checks; - - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) - checks.maxSize = maxSize->get(); - - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) - checks.maxClosureSize = maxClosureSize->get(); - - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { - Strings res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); - res.push_back(j->get()); - } - checks.disallowedRequisites = res; - return res; - } - return {}; - }; - - checks.allowedReferences = get("allowedReferences"); - checks.allowedRequisites = get("allowedRequisites"); - checks.disallowedReferences = get("disallowedReferences"); - checks.disallowedRequisites = get("disallowedRequisites"); - - applyChecks(checks); - } - } - } else { - // legacy non-structured-attributes case - Checks checks; - checks.ignoreSelfRefs = true; - checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); - checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); - checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); - checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); - applyChecks(checks); - } - } -} - - Path DerivationGoal::openLogFile() { logSize = 0; @@ -3722,26 +1146,15 @@ void DerivationGoal::closeLogFile() } -void DerivationGoal::deleteTmpDir(bool force) +bool DerivationGoal::isReadDesc(int fd) { - if (tmpDir != "") { - /* Don't keep temporary directories for builtins because they - might have privileged stuff (like a copy of netrc). */ - if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(tmpDir.c_str(), 0755); - } - else - deletePath(tmpDir); - tmpDir = ""; - } + return fd == hook->builderOut.readSide.get(); } void DerivationGoal::handleChildOutput(int fd, const string & data) { - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) + if (isReadDesc(fd)) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { @@ -3857,22 +1270,6 @@ void DerivationGoal::checkPathValidity() } -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); -} - - -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) -{ - return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); -} - - void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 6dc164922..c85bcd84f 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,7 +2,8 @@ #include "parsed-derivations.hh" #include "lock.hh" -#include "local-store.hh" +#include "store-api.hh" +#include "pathlocks.hh" #include "goal.hh" namespace nix { @@ -79,18 +80,6 @@ struct DerivationGoal : public Goal std::map initialOutputs; - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - /* File descriptor for the log file. */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; @@ -106,79 +95,15 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - /* The build hook. */ std::unique_ptr hook; - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - /* The sort of derivation we are building. */ DerivationType derivationType; - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - typedef void (DerivationGoal::*GoalState)(); GoalState state; - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - /* The final output paths of the build. - For input-addressed derivations, always the precomputed paths @@ -191,11 +116,6 @@ struct DerivationGoal : public Goal BuildMode buildMode; - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - BuildResult result; /* The current round, if we're building multiple times. */ @@ -203,17 +123,6 @@ struct DerivationGoal : public Goal size_t nrRounds; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? 100 : buildUser->getGID(); } - - const static Path homeDir; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; @@ -226,39 +135,13 @@ struct DerivationGoal : public Goal /* The remote machine on which we're building. */ std::string machineName; - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - - friend struct RestrictedStore; - DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - ~DerivationGoal(); - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); + virtual ~DerivationGoal(); void timedOut(Error && ex) override; @@ -280,7 +163,7 @@ struct DerivationGoal : public Goal void closureRepaired(); void inputsRealised(); void tryToBuild(); - void tryLocalBuild(); + virtual void tryLocalBuild(); void buildDone(); void resolvedFinished(); @@ -288,40 +171,11 @@ struct DerivationGoal : public Goal /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - /* Run the builder's process. */ - void runChild(); + virtual int getChildStatus(); /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs(); - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); + virtual void registerOutputs(); /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -329,8 +183,18 @@ struct DerivationGoal : public Goal /* Close the log file. */ void closeLogFile(); - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); + /* Close the read side of the logger pipe. */ + virtual void closeReadPipes(); + + /* Cleanup hooks for buildDone() */ + virtual void cleanupHookFinally(); + virtual void cleanupPreChildKill(); + virtual void cleanupPostChildKill(); + virtual bool cleanupDecideWhetherDiskFull(); + virtual void cleanupPostOutputsRegisteredModeCheck(); + virtual void cleanupPostOutputsRegisteredModeNonCheck(); + + virtual bool isReadDesc(int fd); /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; @@ -347,17 +211,7 @@ struct DerivationGoal : public Goal void checkPathValidity(); /* Forcibly kill the child process, if any. */ - void killChild(); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); + virtual void killChild(); void repairClosure(); @@ -370,4 +224,6 @@ struct DerivationGoal : public Goal StorePathSet exportReferences(const StorePathSet & storePaths); }; +MakeError(NotDeterministic, BuildError); + } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 3a05a022c..01a564aba 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,6 +2,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" +#include "local-store.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 924c69fb7..23ffe740a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,4 @@ -#include "derivation-goal.hh" +#include "local-derivation-goal.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -75,7 +75,6 @@ void handleDiffHook( diffHookOptions.uid = uid; diffHookOptions.gid = gid; diffHookOptions.chdir = "/"; - auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) throw ExecError(diffRes.first, @@ -94,8 +93,9 @@ void handleDiffHook( } } -const Path DerivationGoal::homeDir = "/homeless-shelter"; +const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +#if 0 DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -138,19 +138,20 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation garbage-collected. (See isActiveTempFile() in gc.cc.) */ worker.store.addTempRoot(this->drvPath); } +#endif -DerivationGoal::~DerivationGoal() +LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ + try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { killChild(); } catch (...) { ignoreException(); } try { stopDaemon(); } catch (...) { ignoreException(); } - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { closeLogFile(); } catch (...) { ignoreException(); } } +#if 0 string DerivationGoal::key() { /* Ensure that derivations get built in order of their name, @@ -159,9 +160,10 @@ string DerivationGoal::key() derivation goals (due to "b$"). */ return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } +#endif -inline bool DerivationGoal::needsHashRewrite() +inline bool LocalDerivationGoal::needsHashRewrite() { #if __linux__ return !useChroot; @@ -172,7 +174,15 @@ inline bool DerivationGoal::needsHashRewrite() } -void DerivationGoal::killChild() +LocalStore & LocalDerivationGoal::getLocalStore() +{ + auto p = dynamic_cast(&worker.store); + assert(p); + return *p; +} + + +void LocalDerivationGoal::killChild() { if (pid != -1) { worker.childTerminated(this); @@ -193,17 +203,11 @@ void DerivationGoal::killChild() assert(pid == -1); } - hook.reset(); -} - - -void DerivationGoal::timedOut(Error && ex) -{ - killChild(); - done(BuildResult::TimedOut, ex); + DerivationGoal::killChild(); } +#if 0 void DerivationGoal::work() { (this->*state)(); @@ -695,15 +699,9 @@ void DerivationGoal::tryToBuild() state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); } +#endif -void DerivationGoal::tryLocalBuild() { - /* Make sure that we are allowed to start a build. */ - if (!dynamic_cast(&worker.store)) { - throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); - } +void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); @@ -757,7 +755,6 @@ void DerivationGoal::tryLocalBuild() { started(); } - static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) @@ -785,51 +782,125 @@ static void movePath(const Path & src, const Path & dst) } -void replaceValidPath(const Path & storePath, const Path & tmpPath) +extern void replaceValidPath(const Path & storePath, const Path & tmpPath); + + +int LocalDerivationGoal::getChildStatus() { - /* We can't atomically replace storePath (the original) with - tmpPath (the replacement), so we have to move it out of the - way first. We'd better not be interrupted here, because if - we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); - if (pathExists(storePath)) - movePath(storePath, oldPath); + return hook ? DerivationGoal::getChildStatus() : pid.kill(); +} - try { - movePath(tmpPath, storePath); - } catch (...) { - try { - // attempt to recover - movePath(oldPath, storePath); - } catch (...) { - ignoreException(); - } - throw; - } - - deletePath(oldPath); +void LocalDerivationGoal::closeReadPipes() +{ + if (hook) { + DerivationGoal::closeReadPipes(); + } else + builderOut.readSide = -1; } -MakeError(NotDeterministic, BuildError); +void LocalDerivationGoal::cleanupHookFinally() +{ + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + buildUser.reset(); +} +void LocalDerivationGoal::cleanupPreChildKill() +{ + sandboxMountNamespace = -1; +} + + +void LocalDerivationGoal::cleanupPostChildKill() +{ + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + /* Terminate the recursive Nix daemon. */ + stopDaemon(); +} + + +bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() +{ + bool diskFull = false; + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + { + auto & localStore = getLocalStore(); + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & [_, status] : initialOutputs) { + if (!status.known) continue; + if (buildMode != bmCheck && status.known->isValid()) continue; + auto p = worker.store.printStorePath(status.known->path); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } + + return diskFull; +} + + +void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck() +{ + deleteTmpDir(true); +} + + +void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() +{ + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(worker.store.Store::toRealPath(i.second)); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + cleanupPostOutputsRegisteredModeCheck(); +} + + +#if 0 void DerivationGoal::buildDone() { trace("build done"); - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - Finally releaseBuildUser([&]() { buildUser.reset(); }); + Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - sandboxMountNamespace = -1; + cleanupPreChildKill(); /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, kill it. */ - int status = hook ? hook->pid.kill() : pid.kill(); + int status = getChildStatus(); debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); @@ -840,24 +911,12 @@ void DerivationGoal::buildDone() worker.childTerminated(this); /* Close the read side of the logger pipe. */ - if (hook) { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; - } else - builderOut.readSide = -1; + closeReadPipes(); /* Close the log file. */ closeLogFile(); - /* When running under a build user, make sure that all processes - running under that uid are gone. This is to prevent a - malicious user from leaving behind a process that keeps files - open and modifies them after they have been chown'ed to - root. */ - if (buildUser) buildUser->kill(); - - /* Terminate the recursive Nix daemon. */ - stopDaemon(); + cleanupPostChildKill(); bool diskFull = false; @@ -866,36 +925,7 @@ void DerivationGoal::buildDone() /* Check the exit status. */ if (!statusOk(status)) { - /* Heuristically check whether the build failure may have - been caused by a disk full condition. We have no way - of knowing whether the build actually got an ENOSPC. - So instead, check if the disk is (nearly) full now. If - so, we don't mark this build as a permanent failure. */ -#if HAVE_STATVFS - if (auto localStore = dynamic_cast(&worker.store)) { - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - } -#endif - - deleteTmpDir(false); - - /* Move paths out of the chroot for easier debugging of - build failures. */ - if (useChroot && buildMode == bmNormal) - for (auto & [_, status] : initialOutputs) { - if (!status.known) continue; - if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); - if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); - } + diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", yellowtxt(worker.store.printStorePath(drvPath)), @@ -975,19 +1005,12 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeCheck(); done(BuildResult::Built); return; } - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(worker.store.Store::toRealPath(i.second)); - - /* Delete the chroot (if we were using one). */ - autoDelChroot.reset(); /* this runs the destructor */ - - deleteTmpDir(true); + cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { @@ -1169,15 +1192,17 @@ HookReply DerivationGoal::tryBuildHook() return rpAccept; } +#endif int childEntry(void * arg) { - ((DerivationGoal *) arg)->runChild(); + ((LocalDerivationGoal *) arg)->runChild(); return 1; } +#if 0 StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { StorePathSet paths; @@ -1212,6 +1237,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) return paths; } +#endif static std::once_flag dns_resolve_flag; @@ -1230,7 +1256,7 @@ static void preloadNSS() { } -void linkOrCopy(const Path & from, const Path & to) +static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a @@ -1247,7 +1273,7 @@ void linkOrCopy(const Path & from, const Path & to) } -void DerivationGoal::startBuilder() +void LocalDerivationGoal::startBuilder() { /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) @@ -1285,15 +1311,13 @@ void DerivationGoal::startBuilder() useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } - if (auto localStoreP = dynamic_cast(&worker.store)) { - auto & localStore = *localStoreP; - if (localStore.storeDir != localStore.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif } /* Create a temporary directory where the build will take @@ -1850,7 +1874,7 @@ void DerivationGoal::startBuilder() } -void DerivationGoal::initTmpDir() { +void LocalDerivationGoal::initTmpDir() { /* In a sandbox, for determinism, always use the same temporary directory. */ #if __linux__ @@ -1899,7 +1923,7 @@ void DerivationGoal::initTmpDir() { } -void DerivationGoal::initEnv() +void LocalDerivationGoal::initEnv() { env.clear(); @@ -1960,7 +1984,7 @@ void DerivationGoal::initEnv() static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); -void DerivationGoal::writeStructuredAttrs() +void LocalDerivationGoal::writeStructuredAttrs() { auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; @@ -2079,9 +2103,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo { ref next; - DerivationGoal & goal; + LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) : StoreConfig(params) , LocalFSStoreConfig(params) , RestrictedStoreConfig(params) @@ -2256,15 +2280,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo }; -void DerivationGoal::startDaemon() +void LocalDerivationGoal::startDaemon() { settings.requireExperimentalFeature("recursive-nix"); Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - if (auto localStore = dynamic_cast(&worker.store)) - params["root"] = localStore->rootDir; + params["root"] = getLocalStore().rootDir; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -2322,7 +2345,7 @@ void DerivationGoal::startDaemon() } -void DerivationGoal::stopDaemon() +void LocalDerivationGoal::stopDaemon() { if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) throw SysError("shutting down daemon socket"); @@ -2340,7 +2363,7 @@ void DerivationGoal::stopDaemon() } -void DerivationGoal::addDependency(const StorePath & path) +void LocalDerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; @@ -2397,8 +2420,7 @@ void DerivationGoal::addDependency(const StorePath & path) } } - -void DerivationGoal::chownToBuilder(const Path & path) +void LocalDerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) @@ -2469,7 +2491,7 @@ void setupSeccomp() } -void DerivationGoal::runChild() +void LocalDerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -2977,7 +2999,7 @@ void DerivationGoal::runChild() } -void DerivationGoal::registerOutputs() +void LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2987,14 +3009,8 @@ void DerivationGoal::registerOutputs() floating content-addressed derivations this isn't the case. */ if (hook) { - bool allValid = true; - for (auto & [outputName, outputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!outputPath || !worker.store.isValidPath(*outputPath)) - allValid = false; - else - finalOutputs.insert_or_assign(outputName, *outputPath); - } - if (allValid) return; + DerivationGoal::registerOutputs(); + return; } std::map infos; @@ -3349,10 +3365,7 @@ void DerivationGoal::registerOutputs() } } - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; + auto & localStore = getLocalStore(); if (buildMode == bmCheck) { @@ -3481,10 +3494,7 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ { - auto localStoreP = dynamic_cast(&worker.store); - if (!localStoreP) - throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); - auto & localStore = *localStoreP; + auto & localStore = getLocalStore(); ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { @@ -3513,7 +3523,7 @@ void DerivationGoal::registerOutputs() } -void DerivationGoal::checkOutputs(const std::map & outputs) +void LocalDerivationGoal::checkOutputs(const std::map & outputs) { std::map outputsByPath; for (auto & output : outputs) @@ -3678,6 +3688,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) } +#if 0 Path DerivationGoal::openLogFile() { logSize = 0; @@ -3720,9 +3731,10 @@ void DerivationGoal::closeLogFile() logSink = logFileSink = 0; fdLogFile = -1; } +#endif -void DerivationGoal::deleteTmpDir(bool force) +void LocalDerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { /* Don't keep temporary directories for builtins because they @@ -3738,10 +3750,17 @@ void DerivationGoal::deleteTmpDir(bool force) } +bool LocalDerivationGoal::isReadDesc(int fd) +{ + return (hook && DerivationGoal::isReadDesc(fd)) || + (!hook && fd == builderOut.readSide.get()); +} + + +#if 0 void DerivationGoal::handleChildOutput(int fd, const string & data) { - if ((hook && fd == hook->builderOut.readSide.get()) || - (!hook && fd == builderOut.readSide.get())) + if (isReadDesc(fd)) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { @@ -3855,9 +3874,10 @@ void DerivationGoal::checkPathValidity() } } } +#endif -StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), @@ -3865,7 +3885,7 @@ StorePath DerivationGoal::makeFallbackPath(std::string_view outputName) } -StorePath DerivationGoal::makeFallbackPath(const StorePath & path) +StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), @@ -3873,6 +3893,7 @@ StorePath DerivationGoal::makeFallbackPath(const StorePath & path) } +#if 0 void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; @@ -3897,6 +3918,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional ex) worker.updateProgress(); } +#endif } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 6dc164922..f7994113e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,48 +1,15 @@ #pragma once -#include "parsed-derivations.hh" -#include "lock.hh" +#include "derivation-goal.hh" #include "local-store.hh" -#include "goal.hh" namespace nix { -using std::map; - -struct HookInstance; - -typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; - -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ -enum struct PathStatus { - Corrupt, - Absent, - Valid, -}; - -struct InitialOutputStatus { - StorePath path; - PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ - bool isValid() const { - return status == PathStatus::Valid; - } - /* Merely present, allowed to be corrupt */ - bool isPresent() const { - return status == PathStatus::Corrupt - || status == PathStatus::Valid; - } -}; - -struct InitialOutput { - bool wanted; - Hash outputHash; - std::optional known; -}; - -struct DerivationGoal : public Goal +struct LocalDerivationGoal : public DerivationGoal { + LocalStore & getLocalStore(); + +#if 0 /* Whether to use an on-disk .drv file. */ bool useDerivation; @@ -78,6 +45,7 @@ struct DerivationGoal : public Goal StorePathSet inputPaths; std::map initialOutputs; +#endif /* User selected for running the builder. */ std::unique_ptr buildUser; @@ -91,6 +59,7 @@ struct DerivationGoal : public Goal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; +#if 0 /* File descriptor for the log file. */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; @@ -105,6 +74,7 @@ struct DerivationGoal : public Goal size_t currentLogLinePos = 0; // to handle carriage return std::string currentHookLine; +#endif /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -120,8 +90,10 @@ struct DerivationGoal : public Goal namespace. */ bool usingUserNamespace = true; +#if 0 /* The build hook. */ std::unique_ptr hook; +#endif /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -131,14 +103,18 @@ struct DerivationGoal : public Goal /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; +#if 0 /* The sort of derivation we are building. */ DerivationType derivationType; +#endif /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; +#if 0 typedef void (DerivationGoal::*GoalState)(); GoalState state; +#endif /* Stuff we need to pass to initChild(). */ struct ChrootPath { @@ -179,6 +155,7 @@ struct DerivationGoal : public Goal */ OutputPathMap scratchOutputs; +#if 0 /* The final output paths of the build. - For input-addressed derivations, always the precomputed paths @@ -190,18 +167,21 @@ struct DerivationGoal : public Goal OutputPathMap finalOutputs; BuildMode buildMode; +#endif /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ StorePathSet redirectedBadOutputs; +#if 0 BuildResult result; /* The current round, if we're building multiple times. */ size_t curRound = 1; size_t nrRounds; +#endif /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it @@ -214,6 +194,7 @@ struct DerivationGoal : public Goal const static Path homeDir; +#if 0 std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; @@ -225,6 +206,7 @@ struct DerivationGoal : public Goal /* The remote machine on which we're building. */ std::string machineName; +#endif /* The recursive Nix daemon socket. */ AutoCloseFD daemonSocket; @@ -249,17 +231,14 @@ struct DerivationGoal : public Goal friend struct RestrictedStore; - DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - ~DerivationGoal(); + using DerivationGoal::DerivationGoal; + + virtual ~LocalDerivationGoal() override; /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); +#if 0 void timedOut(Error && ex) override; string key() override; @@ -280,13 +259,16 @@ struct DerivationGoal : public Goal void closureRepaired(); void inputsRealised(); void tryToBuild(); - void tryLocalBuild(); +#endif + void tryLocalBuild() override; +#if 0 void buildDone(); void resolvedFinished(); /* Is the build hook willing to perform the build? */ HookReply tryBuildHook(); +#endif /* Start building a derivation. */ void startBuilder(); @@ -311,27 +293,46 @@ struct DerivationGoal : public Goal /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); + int getChildStatus() override; + /* Run the builder's process. */ void runChild(); /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs(); + void registerOutputs() override; /* Check that an output meets the requirements specified by the 'outputChecks' attribute (or the legacy '{allowed,disallowed}{References,Requisites}' attributes). */ void checkOutputs(const std::map & outputs); +#if 0 /* Open a log file and a pipe to it. */ Path openLogFile(); /* Close the log file. */ void closeLogFile(); +#endif + + /* Close the read side of the logger pipe. */ + void closeReadPipes() override; + + /* Cleanup hooks for buildDone() */ + void cleanupHookFinally() override; + void cleanupPreChildKill() override; + void cleanupPostChildKill() override; + bool cleanupDecideWhetherDiskFull() override; + void cleanupPostOutputsRegisteredModeCheck() override; + void cleanupPostOutputsRegisteredModeNonCheck() override; + + bool isReadDesc(int fd) override; + /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); +#if 0 /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; @@ -345,9 +346,10 @@ struct DerivationGoal : public Goal /* Return the set of (in)valid paths. */ void checkPathValidity(); +#endif /* Forcibly kill the child process, if any. */ - void killChild(); + void killChild() override; /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) @@ -359,6 +361,7 @@ struct DerivationGoal : public Goal rewrites caught everything */ StorePath makeFallbackPath(std::string_view outputName); +#if 0 void repairClosure(); void started(); @@ -368,6 +371,7 @@ struct DerivationGoal : public Goal std::optional ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); +#endif }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2f13aa885..b2223c3b6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,7 +1,7 @@ #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" -#include "derivation-goal.hh" +#include "local-derivation-goal.hh" #include "hook-instance.hh" #include @@ -59,8 +59,10 @@ std::shared_ptr Worker::makeDerivationGoalCommon( std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) + : std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } @@ -68,8 +70,10 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() { - return std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) + : std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 780cc0f07..03bb0218d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -280,7 +280,7 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend struct DerivationGoal; + friend struct LocalDerivationGoal; friend struct SubstitutionGoal; }; From d560311f7643096ce815a7c655a077621abb7d1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 15:31:15 +0000 Subject: [PATCH 160/725] Remove temporary `#if 0...#endif` from previous commit --- src/libstore/build/local-derivation-goal.cc | 1090 ------------------- src/libstore/build/local-derivation-goal.hh | 175 +-- 2 files changed, 1 insertion(+), 1264 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 23ffe740a..3a0616864 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -95,51 +95,6 @@ void handleDiffHook( const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; -#if 0 -DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(true) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &DerivationGoal::getDerivation; - name = fmt( - "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); -} - - -DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) - , useDerivation(false) - , drvPath(drvPath) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - this->drv = std::make_unique(drv); - - state = &DerivationGoal::haveDerivation; - name = fmt( - "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); - trace("created"); - - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); - worker.updateProgress(); - - /* Prevent the .chroot directory from being - garbage-collected. (See isActiveTempFile() in gc.cc.) */ - worker.store.addTempRoot(this->drvPath); -} -#endif - LocalDerivationGoal::~LocalDerivationGoal() { @@ -151,18 +106,6 @@ LocalDerivationGoal::~LocalDerivationGoal() } -#if 0 -string DerivationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); -} -#endif - - inline bool LocalDerivationGoal::needsHashRewrite() { #if __linux__ @@ -207,500 +150,6 @@ void LocalDerivationGoal::killChild() } -#if 0 -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; -} - - -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); - - state = &DerivationGoal::loadDerivation; -} - - -void DerivationGoal::loadDerivation() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; - } - - /* `drvPath' should already be a root, but let's be on the safe - side: if the user forgot to make it a root, we wouldn't want - things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); - - assert(worker.store.isValidPath(drvPath)); - - /* Get the derivation. */ - drv = std::make_unique(worker.store.derivationFromPath(drvPath)); - - haveDerivation(); -} - - -void DerivationGoal::haveDerivation() -{ - trace("have derivation"); - - if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); - - retrySubstitution = false; - - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - worker.store.addTempRoot(*i.second.second); - - auto outputHashes = staticOutputHashes(worker.store, *drv); - for (auto &[outputName, outputHash] : outputHashes) - initialOutputs.insert({ - outputName, - InitialOutput{ - .wanted = true, // Will be refined later - .outputHash = outputHash - } - }); - - /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - - /* If they are all valid, then we're done. */ - if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); - return; - } - - parsedDrv = std::make_unique(drvPath, *drv); - - - /* We are first going to try to create the invalid output paths - through substitutes. If that doesn't work, we'll build - them. */ - if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(upcast_goal(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; -} - - -void DerivationGoal::outputsSubstitutionTried() -{ - trace("all outputs substituted (maybe)"); - - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); - return; - } - - /* If the substitutes form an incomplete closure, then we should - build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. - - If the nrIncompleteClosure != nrFailed, we have another issue as well. - In particular, it may be the case that the hole in the closure is - an output of the current derivation, which causes a loop if retried. - */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; - - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - - if (needRestart) { - needRestart = false; - haveDerivation(); - return; - } - - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } - - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); - return; - } - if (buildMode == bmRepair && nrInvalid == 0) { - repairClosure(); - return; - } - if (buildMode == bmCheck && nrInvalid > 0) - throw Error("some outputs of '%s' are not valid, so checking is not possible", - worker.store.printStorePath(drvPath)); - - /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); -} - -/* At least one of the output paths could not be - produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() -{ - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - - /* The inputs must be built before we can build this goal. */ - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - - for (auto & i : drv->inputSrcs) { - if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) - throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); - } - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::repairClosure() -{ - /* If we're repairing, we now know that our own outputs are valid. - Now check whether the other paths in the outputs closure are - good. If not, then start derivation goals for the derivations - that produced those outputs. */ - - /* Get the output closure. */ - auto outputs = queryDerivationOutputMap(); - StorePathSet outputClosure; - for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; - worker.store.computeFSClosure(i.second, outputClosure); - } - - /* Filter out our own outputs (which we have already checked). */ - for (auto & i : outputs) - outputClosure.erase(i.second); - - /* Get all dependencies of this derivation so that we know which - derivation is responsible for which path in the output - closure. */ - StorePathSet inputClosure; - if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map outputsToDrv; - for (auto & i : inputClosure) - if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); - for (auto & j : depOutputs) - if (j.second) - outputsToDrv.insert_or_assign(*j.second, i); - } - - /* Check each path (slow!). */ - for (auto & i : outputClosure) { - if (worker.pathContentsGood(i)) continue; - printError( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - auto drvPath2 = outputsToDrv.find(i); - if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); - else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); - } - - if (waitees.empty()) { - done(BuildResult::AlreadyValid); - return; - } - - state = &DerivationGoal::closureRepaired; -} - - -void DerivationGoal::closureRepaired() -{ - trace("closure repaired"); - if (nrFailed > 0) - throw Error("some paths in the output closure of derivation '%s' could not be repaired", - worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - if (!useDerivation) - throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( - "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); - return; - } - - if (retrySubstitution) { - haveDerivation(); - return; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - if (useDerivation) { - auto & fullDrv = *dynamic_cast(drv.get()); - - if (settings.isExperimentalFeatureEnabled("ca-derivations") && - ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed)) { - /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); - assert(attempt); - Derivation drvResolved { *std::move(attempt) }; - - auto pathResolved = writeDerivation(worker.store, drvResolved); - resolvedDrv = drvResolved; - - auto msg = fmt("Resolved derivation: '%s' -> '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved)); - act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, - Logger::Fields { - worker.store.printStorePath(drvPath), - worker.store.printStorePath(pathResolved), - }); - - auto resolvedGoal = worker.makeDerivationGoal( - pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); - - state = &DerivationGoal::resolvedFinished; - return; - } - - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - } - } - } - - /* Second, the input sources. */ - worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - - debug("added input paths %s", worker.store.showPaths(inputPaths)); - - /* What type of derivation are we building? */ - derivationType = drv->type(); - - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - - result = BuildResult(); -} - -void DerivationGoal::started() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - if (dynamic_cast(&worker.store)) - /* If we aren't a local store, we might need to use the local store as - a build remote, but that would cause a deadlock. */ - /* FIXME: Make it so we can use ourselves as a build remote even if we - are the local store (separate locking for building vs scheduling? */ - /* FIXME: find some way to lock for scheduling for the other stores so - a forking daemon with --store still won't farm out redundant builds. - */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - - if (!outputLocks.lockPaths(lockFiles, "", false)) { - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return; - } - - actLock.reset(); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); - return; - } - - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } - - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - result.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return; - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - } - } - - actLock.reset(); - - state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); -} -#endif - void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { @@ -887,314 +336,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -#if 0 -void DerivationGoal::buildDone() -{ - trace("build done"); - - Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); - - cleanupPreChildKill(); - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = getChildStatus(); - - debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - - result.timesBuilt++; - result.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - - /* Close the read side of the logger pipe. */ - closeReadPipes(); - - /* Close the log file. */ - closeLogFile(); - - cleanupPostChildKill(); - - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= cleanupDecideWhetherDiskFull(); - - auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), - statusToString(status)); - - if (!logger->isVerbose() && !logTail.empty()) { - msg += fmt(";\nlast %d log lines:\n", logTail.size()); - for (auto & line : logTail) { - msg += "> "; - msg += line; - msg += "\n"; - } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", - worker.store.printStorePath(drvPath)); - } - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - registerOutputs(); - - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (std::string_view data) override { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } - - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built); - return; - } - - cleanupPostOutputsRegisteredModeNonCheck(); - - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - } catch (BuildError & e) { - outputLocks.unlock(); - - BuildResult::Status st = BuildResult::MiscFailure; - - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) - st = BuildResult::TimedOut; - - else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - } - - else { - st = - dynamic_cast(&e) ? BuildResult::NotDeterministic : - statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : - BuildResult::PermanentFailure; - } - - done(st, e); - return; - } - - done(BuildResult::Built); -} - -void DerivationGoal::resolvedFinished() { - assert(resolvedDrv); - - auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); - - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv->outputNames(); - - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} - ); - // We've just built it, but maybe the build failed, in which case the - // realisation won't be there - if (realisation) { - auto newRealisation = *realisation; - newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; - worker.store.registerDrvOutput(newRealisation); - } else { - // If we don't have a realisation, then it must mean that something - // failed when building the resolved drv - assert(!result.success()); - } - } - - // This is potentially a bit fishy in terms of error reporting. Not sure - // how to do it in a cleaner way - amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); -} - -HookReply DerivationGoal::tryBuildHook() -{ - if (!worker.tryBuildHook || !useDerivation) return rpDecline; - - if (!worker.hook) - worker.hook = std::make_unique(); - - try { - - /* Send the request to the hook. */ - worker.hook->sink - << "try" - << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) - << drv->platform - << worker.store.printStorePath(drvPath) - << parsedDrv->getRequiredSystemFeatures(); - worker.hook->sink.flush(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. */ - string reply; - while (true) { - auto s = [&]() { - try { - return readLine(worker.hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the response from the build hook"); - throw e; - } - }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) - ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - else { - s += "\n"; - writeToStderr(s); - } - } - - debug("hook reply is '%1%'", reply); - - if (reply == "decline") - return rpDecline; - else if (reply == "decline-permanently") { - worker.tryBuildHook = false; - worker.hook = 0; - return rpDecline; - } - else if (reply == "postpone") - return rpPostpone; - else if (reply != "accept") - throw Error("bad hook reply '%s'", reply); - - } catch (SysError & e) { - if (e.errNo == EPIPE) { - printError( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); - worker.hook = 0; - return rpDecline; - } else - throw; - } - - hook = std::move(worker.hook); - - try { - machineName = readLine(hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the machine name from the build hook"); - throw e; - } - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StringSet missingOutputs; - for (auto & [outputName, status] : initialOutputs) { - // XXX: Does this include known CA outputs? - if (buildMode != bmCheck && status.known && status.known->isValid()) continue; - missingOutputs.insert(outputName); - } - worker_proto::write(worker.store, hook->sink, missingOutputs); - } - - hook->sink = FdSink(); - hook->toHook.writeSide = -1; - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); - - set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - - return rpAccept; -} -#endif - - int childEntry(void * arg) { ((LocalDerivationGoal *) arg)->runChild(); @@ -1202,43 +343,6 @@ int childEntry(void * arg) } -#if 0 -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} -#endif - static std::once_flag dns_resolve_flag; static void preloadNSS() { @@ -3688,52 +2792,6 @@ void LocalDerivationGoal::checkOutputs(const std::map & out } -#if 0 -Path DerivationGoal::openLogFile() -{ - logSize = 0; - - if (!settings.keepLog) return ""; - - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); - - /* Create a log file. */ - Path logDir; - if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; - else - logDir = settings.nixLogDir; - Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); - createDirs(dir); - - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), - settings.compressLog ? ".bz2" : ""); - - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); - - logFileSink = std::make_shared(fdLogFile.get()); - - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); - else - logSink = logFileSink; - - return logFileName; -} - - -void DerivationGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) logSink2->finish(); - if (logFileSink) logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile = -1; -} -#endif - - void LocalDerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { @@ -3757,126 +2815,6 @@ bool LocalDerivationGoal::isReadDesc(int fd) } -#if 0 -void DerivationGoal::handleChildOutput(int fd, const string & data) -{ - if (isReadDesc(fd)) - { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - done( - BuildResult::LogLimitExceeded, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) (*logSink)(data); - } - - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); - currentHookLine.clear(); - } else - currentHookLine += c; - } -} - - -void DerivationGoal::handleEOF(int fd) -{ - if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::flushLine() -{ - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; -} - - -std::map> DerivationGoal::queryPartialDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - std::map> res; - for (auto & [name, output] : drv->outputs) - res.insert_or_assign(name, output.path(worker.store, drv->name, name)); - return res; - } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); - } -} - -OutputPathMap DerivationGoal::queryDerivationOutputMap() -{ - if (!useDerivation || drv->type() != DerivationType::CAFloating) { - OutputPathMap res; - for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) - res.insert_or_assign(name, *output.second); - return res; - } else { - return worker.store.queryDerivationOutputMap(drvPath); - } -} - - -void DerivationGoal::checkPathValidity() -{ - bool checkHash = buildMode == bmRepair; - for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput & info = initialOutputs.at(i.first); - info.wanted = wantOutput(i.first, wantedOutputs); - if (i.second) { - auto outputPath = *i.second; - info.known = { - .path = outputPath, - .status = !worker.store.isValidPath(outputPath) - ? PathStatus::Absent - : !checkHash || worker.pathContentsGood(outputPath) - ? PathStatus::Valid - : PathStatus::Corrupt, - }; - } - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (auto real = worker.store.queryRealisation( - DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { - info.known = { - .path = real->outPath, - .status = PathStatus::Valid, - }; - } - } - } -} -#endif - - StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) { return worker.store.makeStorePath( @@ -3893,32 +2831,4 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) } -#if 0 -void DerivationGoal::done(BuildResult::Status status, std::optional ex) -{ - result.status = status; - if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) - worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; - - mcExpectedBuilds.reset(); - mcRunningBuilds.reset(); - - if (result.success()) { - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } - - worker.updateProgress(); -} -#endif - - } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f7994113e..a2b386a72 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -9,44 +9,6 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); -#if 0 - /* Whether to use an on-disk .drv file. */ - bool useDerivation; - - /* The path of the derivation. */ - StorePath drvPath; - - /* The path of the corresponding resolved derivation */ - std::optional resolvedDrv; - - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; - - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; - - /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; - - /* The derivation stored at drvPath. */ - std::unique_ptr drv; - - std::unique_ptr parsedDrv; - - /* The remainder is state held during the build. */ - - /* Locks on (fixed) output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - StorePathSet inputPaths; - - std::map initialOutputs; -#endif - /* User selected for running the builder. */ std::unique_ptr buildUser; @@ -59,23 +21,6 @@ struct LocalDerivationGoal : public DerivationGoal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; -#if 0 - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /* Number of bytes received from the builder's stdout/stderr. */ - unsigned long logSize; - - /* The most recent log lines. */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; -#endif - /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -90,11 +35,6 @@ struct LocalDerivationGoal : public DerivationGoal namespace. */ bool usingUserNamespace = true; -#if 0 - /* The build hook. */ - std::unique_ptr hook; -#endif - /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -103,19 +43,9 @@ struct LocalDerivationGoal : public DerivationGoal /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; -#if 0 - /* The sort of derivation we are building. */ - DerivationType derivationType; -#endif - /* Whether to run the build in a private network namespace. */ bool privateNetwork = false; -#if 0 - typedef void (DerivationGoal::*GoalState)(); - GoalState state; -#endif - /* Stuff we need to pass to initChild(). */ struct ChrootPath { Path source; @@ -155,34 +85,11 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; -#if 0 - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; -#endif - /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ StorePathSet redirectedBadOutputs; -#if 0 - BuildResult result; - - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; -#endif - /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same @@ -194,20 +101,6 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; -#if 0 - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; - - std::unique_ptr act; - - /* Activity that denotes waiting for a lock. */ - std::unique_ptr actLock; - - std::map builderActivities; - - /* The remote machine on which we're building. */ - std::string machineName; -#endif - /* The recursive Nix daemon socket. */ AutoCloseFD daemonSocket; @@ -238,37 +131,8 @@ struct LocalDerivationGoal : public DerivationGoal /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); -#if 0 - void timedOut(Error && ex) override; - - string key() override; - - void work() override; - - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); - - BuildResult getResult() { return result; } - - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); -#endif + /* The additional states. */ void tryLocalBuild() override; -#if 0 - void buildDone(); - - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); -#endif /* Start building a derivation. */ void startBuilder(); @@ -307,14 +171,6 @@ struct LocalDerivationGoal : public DerivationGoal '{allowed,disallowed}{References,Requisites}' attributes). */ void checkOutputs(const std::map & outputs); -#if 0 - /* Open a log file and a pipe to it. */ - Path openLogFile(); - - /* Close the log file. */ - void closeLogFile(); -#endif - /* Close the read side of the logger pipe. */ void closeReadPipes() override; @@ -328,26 +184,9 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); -#if 0 - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; - void handleEOF(int fd) override; - void flushLine(); - - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ - std::map> queryPartialDerivationOutputMap(); - OutputPathMap queryDerivationOutputMap(); - - /* Return the set of (in)valid paths. */ - void checkPathValidity(); -#endif - /* Forcibly kill the child process, if any. */ void killChild() override; @@ -360,18 +199,6 @@ struct LocalDerivationGoal : public DerivationGoal /* FIXME add option to randomize, so we can audit whether our rewrites caught everything */ StorePath makeFallbackPath(std::string_view outputName); - -#if 0 - void repairClosure(); - - void started(); - - void done( - BuildResult::Status status, - std::optional ex = {}); - - StorePathSet exportReferences(const StorePathSet & storePaths); -#endif }; } From 553b79f8c980fde70fe186ee4980b2d12e27d756 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Feb 2021 17:38:38 +0000 Subject: [PATCH 161/725] Remove unused `redirectedBadOutputs` --- src/libstore/build/local-derivation-goal.cc | 4 ---- src/libstore/build/local-derivation-goal.hh | 5 ----- 2 files changed, 9 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3a0616864..9c2f1dda6 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -460,10 +460,6 @@ void LocalDerivationGoal::startBuilder() makeFallbackPath(status.known->path); scratchOutputs.insert_or_assign(outputName, scratchPath); - /* A non-removed corrupted path needs to be stored here, too */ - if (buildMode == bmRepair && !status.known->isValid()) - redirectedBadOutputs.insert(status.known->path); - /* Substitute output placeholders with the scratch output paths. We'll use during the build. */ inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index a2b386a72..4bbf27a1b 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -85,11 +85,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /* If we're repairing without a chroot, there may be outputs that - are valid but corrupt. So we redirect these outputs to - temporary paths. */ - StorePathSet redirectedBadOutputs; - /* Path registration info from the previous round, if we're building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same From 5b42e5b1771061de50575b33eeeda56f40f216f2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Feb 2021 16:29:19 +0000 Subject: [PATCH 162/725] Restore now-working build-remote-content-addressed-fixed test This was - Added in dbf96e10ecc75410c9db798f208f8a8310842a4f. - Commented out in 07975979aae4e7729ae13ffeb7390d07d71ad4bd, which I believe only reached master by mistake. - Deleted in c32168c9bc161e0c9cea027853895971699510cb, when `tests/build-hook-ca.nix` was reused for a new test. But the test works, and we ought to have it. --- tests/build-hook-ca-fixed.nix | 56 +++++++++++++++++++ ...hook-ca.nix => build-hook-ca-floating.nix} | 0 tests/build-remote-content-addressed-fixed.sh | 5 ++ ...build-remote-content-addressed-floating.sh | 2 +- tests/local.mk | 1 + 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/build-hook-ca-fixed.nix rename tests/{build-hook-ca.nix => build-hook-ca-floating.nix} (100%) create mode 100644 tests/build-remote-content-addressed-fixed.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix new file mode 100644 index 000000000..ec7171ac9 --- /dev/null +++ b/tests/build-hook-ca-fixed.nix @@ -0,0 +1,56 @@ +{ busybox }: + +with import ./config.nix; + +let + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo FOO > $out"; + requiredSystemFeatures = ["foo"]; + outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; + }; + + input2 = mkDerivation { + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo BAR > $out"; + requiredSystemFeatures = ["bar"]; + outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + read x < ${input2} + echo $x BAZ > $out + ''; + requiredSystemFeatures = ["baz"]; + outputHash = "sha256-daKAcPp/+BYMQsVi/YYMlCKoNAxCNDsaivwSHgQqD2s="; + }; + +in + + mkDerivation { + shell = busybox; + name = "build-remote"; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + outputHash = "sha256-5SxbkUw6xe2l9TE1uwCvTtTDysD1vhRor38OtDF0LqQ="; + } diff --git a/tests/build-hook-ca.nix b/tests/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca.nix rename to tests/build-hook-ca-floating.nix diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/build-remote-content-addressed-fixed.sh new file mode 100644 index 000000000..ae7441591 --- /dev/null +++ b/tests/build-remote-content-addressed-fixed.sh @@ -0,0 +1,5 @@ +source common.sh + +file=build-hook-ca-fixed.nix + +source build-remote.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index cbb75729b..7447d92bd 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -1,6 +1,6 @@ source common.sh -file=build-hook-ca.nix +file=build-hook-ca-floating.nix sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf diff --git a/tests/local.mk b/tests/local.mk index 7deea9ac1..4d970d5e4 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ ssh-relay.sh \ nar-access.sh \ From 12ec962dd8a6d8058ba11e517d74f6a07b3dc903 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 25 Feb 2021 16:12:51 -0600 Subject: [PATCH 163/725] simplify changing cachix cache for install tests - convert cachix cache name from an env into a secret so it (along with the token/key) can be set once per fork - use CACHIX_AUTH_TOKEN in addition to CACHIX_SIGNING_KEY; it looks like cachix will try signing key first, then auth token. --- .github/workflows/test.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bde6106e0..2531a7d35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,52 +8,62 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci + steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v12 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v8 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) - installer: - if: github.event_name == 'push' - needs: tests + check_cachix: + name: Cachix secret present for installer tests + runs-on: ubuntu-latest + outputs: + secret: ${{ steps.secret.outputs.secret }} + steps: + - name: Check for Cachix secret + id: secret + env: + _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} + run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + installer: + needs: [tests, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' runs-on: ubuntu-latest - env: - CACHIX_NAME: nix-ci outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v12 - uses: cachix/cachix-action@v8 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - id: prepare-installer run: scripts/prepare-installer-for-github-actions installer_test: - if: github.event_name == 'push' - needs: installer + needs: [installer, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci steps: - uses: actions/checkout@v2.3.4 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@master with: install_url: '${{needs.installer.outputs.installerURL}}' - install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve' + install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - run: nix-instantiate -E 'builtins.currentTime' --eval - \ No newline at end of file From bd0b0f9ab7655553f64f158d5d9a9445f5604abd Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 21:48:41 +0000 Subject: [PATCH 164/725] mk: add support for CPPFLAGS --- mk/lib.mk | 1 + mk/patterns.mk | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mk/lib.mk b/mk/lib.mk index a09ebaa97..6b92136cd 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -153,4 +153,5 @@ endif @echo " CFLAGS: Flags for the C compiler" @echo " CXX ($(CXX)): C++ compiler to be used" @echo " CXXFLAGS: Flags for the C++ compiler" + @echo " CPPFLAGS: C preprocessor flags, used for both CC and CXX" @$(print-var-help) diff --git a/mk/patterns.mk b/mk/patterns.mk index 7319f4cdd..86a724806 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,11 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP From 7241fdc3d2386d256ca8870ca955b498d0ac2ff7 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 22:06:06 +0000 Subject: [PATCH 165/725] Properly propagate libseccomp linker flags --- Makefile.config.in | 1 + src/libstore/local.mk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index 9d0500e48..3c1f01d1e 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -17,6 +17,7 @@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ +LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SHELL = @bash@ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 03c4351ac..cf0933705 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -28,7 +28,7 @@ ifeq ($(OS), SunOS) endif ifeq ($(HAVE_SECCOMP), 1) - libstore_LDFLAGS += -lseccomp + libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif libstore_CXXFLAGS += \ From 2d7917f035c7396e87546b130317a2e5234afa36 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 26 Feb 2021 21:42:51 +0000 Subject: [PATCH 166/725] Revert "Add support for building JARs from Java sources" This reverts commit 259086de841d155f7951c2cc50f799a4631aa512. --- mk/jars.mk | 36 ------------------------------------ mk/lib.mk | 12 +----------- mk/tracing.mk | 2 -- 3 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 mk/jars.mk diff --git a/mk/jars.mk b/mk/jars.mk deleted file mode 100644 index c8513e664..000000000 --- a/mk/jars.mk +++ /dev/null @@ -1,36 +0,0 @@ -define build-jar - - $(1)_NAME ?= $(1) - - _d := $$(strip $$($(1)_DIR)) - - $(1)_PATH := $$(_d)/$$($(1)_NAME).jar - - $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp - - _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH)) - - $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER) - @rm -rf $$($(1)_TMPDIR) - @mkdir -p $$($(1)_TMPDIR) - $$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \ - $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \ - -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH" - @echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest - $$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) . - @rm $$($(1)_PATH).manifest - @rm -rf $$($(1)_TMPDIR) - - $(1)_INSTALL_DIR ?= $$(jardir) - - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar - - $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644)) - - install: $$($(1)_INSTALL_PATH) - - jars-list += $$($(1)_PATH) - - clean-files += $$($(1)_PATH) - -endef diff --git a/mk/lib.mk b/mk/lib.mk index a09ebaa97..6a1c465b6 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -31,7 +31,6 @@ libdir ?= $(prefix)/lib bindir ?= $(prefix)/bin libexecdir ?= $(prefix)/libexec datadir ?= $(prefix)/share -jardir ?= $(datadir)/java localstatedir ?= $(prefix)/var sysconfdir ?= $(prefix)/etc mandir ?= $(prefix)/share/man @@ -74,7 +73,6 @@ BUILD_DEBUG ?= 1 ifeq ($(BUILD_DEBUG), 1) GLOBAL_CFLAGS += -g GLOBAL_CXXFLAGS += -g - GLOBAL_JAVACFLAGS += -g endif @@ -84,7 +82,6 @@ include mk/clean.mk include mk/install.mk include mk/libraries.mk include mk/programs.mk -include mk/jars.mk include mk/patterns.mk include mk/templates.mk include mk/tests.mk @@ -102,7 +99,6 @@ $(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) # Instantiate stuff. $(foreach lib, $(libraries), $(eval $(call build-library,$(lib)))) $(foreach prog, $(programs), $(eval $(call build-program,$(prog)))) -$(foreach jar, $(jars), $(eval $(call build-jar,$(jar)))) $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir)))) $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) @@ -113,7 +109,7 @@ $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/ .PHONY: default all man help -all: $(programs-list) $(libs-list) $(jars-list) $(man-pages) +all: $(programs-list) $(libs-list) $(man-pages) man: $(man-pages) @@ -137,12 +133,6 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done -endif -ifdef jars-list - @echo "" - @echo "The following JARs can be built:" - @echo "" - @for i in $(jars-list); do echo " $$i"; done endif @echo "" @echo "The following variables control the build:" diff --git a/mk/tracing.mk b/mk/tracing.mk index 54c77ab60..1fc5573d7 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -8,8 +8,6 @@ ifeq ($(V), 0) trace-ld = @echo " LD " $@; trace-ar = @echo " AR " $@; trace-install = @echo " INST " $@; - trace-javac = @echo " JAVAC " $@; - trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; From ae1441e5488a0e1608851b329358eb390a08ac27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 27 Feb 2021 05:23:14 +0000 Subject: [PATCH 167/725] Fix testing fixed-output derivations in double sandboxes What happened was that Nix was trying to unconditionally mount these paths in fixed-output derivations, but since the outer derivation was pure, those paths did not exist. The solution is to only mount those paths when they exist. --- src/libstore/build/local-derivation-goal.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c2f1dda6..90731d98d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -287,7 +287,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - { + { auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; @@ -297,7 +297,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (statvfs(tmpDir.c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; - } + } #endif deleteTmpDir(false); @@ -1703,18 +1703,18 @@ void LocalDerivationGoal::runChild() network, so give them access to /etc/resolv.conf and so on. */ if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + if (pathExists(path)) + ss.push_back(path); } for (auto & i : ss) dirsInChroot.emplace(i, i); From 4bbd80c5366711b8f1b5ad108ba22206d3bee783 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Feb 2021 21:50:50 +0000 Subject: [PATCH 168/725] Throw error for derivation goal with bogus wanted output --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c29237f5c..530f8829a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1243,9 +1243,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; + auto wantedOutputsLeft = wantedOutputs; for (auto & i : queryPartialDerivationOutputMap()) { InitialOutput & info = initialOutputs.at(i.first); info.wanted = wantOutput(i.first, wantedOutputs); + if (info.wanted) + wantedOutputsLeft.erase(i.first); if (i.second) { auto outputPath = *i.second; info.known = { @@ -1267,6 +1270,11 @@ void DerivationGoal::checkPathValidity() } } } + // If we requested all the outputs via the empty set, we are always fine. + // If we requested specific elements, the loop above removes all the valid + // ones, so any that are left must be invalid. + if (!wantedOutputsLeft.empty()) + throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft)); } From 259d6778efd865ccd3b5fbf4f3a29002a7d58d93 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 16:04:18 +0100 Subject: [PATCH 169/725] Move the CA tests to a sub-directory Requires a slight update to the test infra to work properly, but having the possibility to group tests that way makes the whole thing quite cleaner imho --- mk/run_test.sh | 2 +- mk/tests.mk | 2 +- tests/{content-addressed.sh => ca/build.sh} | 2 ++ tests/ca/common.sh | 1 + tests/{ => ca}/content-addressed.nix | 2 +- tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} | 0 tests/common.sh.in | 2 +- tests/local.mk | 6 +++--- 8 files changed, 10 insertions(+), 7 deletions(-) rename tests/{content-addressed.sh => ca/build.sh} (98%) create mode 100644 tests/ca/common.sh rename tests/{ => ca}/content-addressed.nix (98%) rename tests/{nix-copy-content-addressed.sh => ca/nix-copy.sh} (100%) diff --git a/mk/run_test.sh b/mk/run_test.sh index 6af5b070a..3783d3bf7 100755 --- a/mk/run_test.sh +++ b/mk/run_test.sh @@ -14,7 +14,7 @@ if [ -t 1 ]; then yellow="" normal="" fi -(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) +(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" status=$? if [ $status -eq 0 ]; then diff --git a/mk/tests.mk b/mk/tests.mk index c1e140bac..21bdc5748 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null + @env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null endef diff --git a/tests/content-addressed.sh b/tests/ca/build.sh similarity index 98% rename from tests/content-addressed.sh rename to tests/ca/build.sh index 7e32e1f28..35bf1dcf7 100644 --- a/tests/content-addressed.sh +++ b/tests/ca/build.sh @@ -61,7 +61,9 @@ testNixCommand () { # Disabled until we have it properly working # testRemoteCache +clearStore testDeterministicCA +clearStore testCutoff testGC testNixCommand diff --git a/tests/ca/common.sh b/tests/ca/common.sh new file mode 100644 index 000000000..e083d873c --- /dev/null +++ b/tests/ca/common.sh @@ -0,0 +1 @@ +source ../common.sh diff --git a/tests/content-addressed.nix b/tests/ca/content-addressed.nix similarity index 98% rename from tests/content-addressed.nix rename to tests/ca/content-addressed.nix index 61079176f..e5b1c4de3 100644 --- a/tests/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ../config.nix; { seed ? 0 }: # A simple content-addressed derivation. diff --git a/tests/nix-copy-content-addressed.sh b/tests/ca/nix-copy.sh similarity index 100% rename from tests/nix-copy-content-addressed.sh rename to tests/ca/nix-copy.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index e3bcab507..de44a4da4 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,7 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket +export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then diff --git a/tests/local.mk b/tests/local.mk index 7deea9ac1..07cfd7a50 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -38,10 +38,10 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ - content-addressed.sh \ - nix-copy-content-addressed.sh \ build.sh \ - compute-levels.sh + compute-levels.sh \ + ca/build.sh \ + ca/nix-copy.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 5d1c05b07561c841c68eb3ff9698ce9d2355fe41 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 13:47:06 +0100 Subject: [PATCH 170/725] SubstitutionGoal -> PathSubstitutionGoal To prepare for the upcoming DrvOutputSubstitutionGoal --- src/libstore/build/derivation-goal.cc | 8 +++---- src/libstore/build/entry-points.cc | 8 +++---- src/libstore/build/substitution-goal.cc | 32 ++++++++++++------------- src/libstore/build/substitution-goal.hh | 9 +++---- src/libstore/build/worker.cc | 12 +++++----- src/libstore/build/worker.hh | 12 +++++----- src/libstore/local-store.hh | 2 ++ 7 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c29237f5c..7b97e575a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -170,7 +170,7 @@ void DerivationGoal::getDerivation() return; } - addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); state = &DerivationGoal::loadDerivation; } @@ -253,7 +253,7 @@ void DerivationGoal::haveDerivation() /* Nothing to wait for; tail call */ return DerivationGoal::gaveUpOnSubstitution(); } - addWaitee(upcast_goal(worker.makeSubstitutionGoal( + addWaitee(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)))); @@ -337,7 +337,7 @@ void DerivationGoal::gaveUpOnSubstitution() if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -388,7 +388,7 @@ void DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 01a564aba..686364440 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector & drvPaths, Build if (path.path.isDerivation()) goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); @@ -31,7 +31,7 @@ void Store::buildPaths(const std::vector & drvPaths, Build } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -90,7 +90,7 @@ void Store::ensurePath(const StorePath & path) if (isValidPath(path)) return; Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; worker.run(goals); @@ -108,7 +108,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index c4b0de78d..5d88b8758 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -5,20 +5,20 @@ namespace nix { -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) , ca(ca) { - state = &SubstitutionGoal::init; + state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); } -SubstitutionGoal::~SubstitutionGoal() +PathSubstitutionGoal::~PathSubstitutionGoal() { try { if (thr.joinable()) { @@ -32,13 +32,13 @@ SubstitutionGoal::~SubstitutionGoal() } -void SubstitutionGoal::work() +void PathSubstitutionGoal::work() { (this->*state)(); } -void SubstitutionGoal::init() +void PathSubstitutionGoal::init() { trace("init"); @@ -59,7 +59,7 @@ void SubstitutionGoal::init() } -void SubstitutionGoal::tryNext() +void PathSubstitutionGoal::tryNext() { trace("trying next substituter"); @@ -154,16 +154,16 @@ void SubstitutionGoal::tryNext() paths referenced by this one. */ for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); else - state = &SubstitutionGoal::referencesValid; + state = &PathSubstitutionGoal::referencesValid; } -void SubstitutionGoal::referencesValid() +void PathSubstitutionGoal::referencesValid() { trace("all references realised"); @@ -177,12 +177,12 @@ void SubstitutionGoal::referencesValid() if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &SubstitutionGoal::tryToRun; + state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); } -void SubstitutionGoal::tryToRun() +void PathSubstitutionGoal::tryToRun() { trace("trying to run"); @@ -221,11 +221,11 @@ void SubstitutionGoal::tryToRun() worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - state = &SubstitutionGoal::finished; + state = &PathSubstitutionGoal::finished; } -void SubstitutionGoal::finished() +void PathSubstitutionGoal::finished() { trace("substitute finished"); @@ -249,7 +249,7 @@ void SubstitutionGoal::finished() } /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; + state = &PathSubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } @@ -278,12 +278,12 @@ void SubstitutionGoal::finished() } -void SubstitutionGoal::handleChildOutput(int fd, const string & data) +void PathSubstitutionGoal::handleChildOutput(int fd, const string & data) { } -void SubstitutionGoal::handleEOF(int fd) +void PathSubstitutionGoal::handleEOF(int fd) { if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index dee2cecbf..3b3cb7e32 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,7 +8,7 @@ namespace nix { class Worker; -struct SubstitutionGoal : public Goal +struct PathSubstitutionGoal : public Goal { /* The store path that should be realised through a substitute. */ StorePath storePath; @@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (SubstitutionGoal::*GoalState)(); + typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; /* Content address for recomputing store path */ std::optional ca; - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); +public: + PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b2223c3b6..619b1d69c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -78,12 +78,12 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - std::weak_ptr & goal_weak = substitutionGoals[path]; + std::weak_ptr & goal_weak = substitutionGoals[path]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; wakeUp(goal); } @@ -109,7 +109,7 @@ void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); else assert(false); @@ -217,7 +217,7 @@ void Worker::run(const Goals & _topGoals) topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { topPaths.push_back({goal->drvPath, goal->wantedOutputs}); - } else if (auto goal = dynamic_cast(i.get())) { + } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back({goal->storePath}); } } @@ -471,7 +471,7 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 82e711191..42acf8542 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -12,18 +12,18 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; -struct SubstitutionGoal; +struct PathSubstitutionGoal; /* Workaround for not being able to declare a something like - class SubstitutionGoal : public Goal; + class PathSubstitutionGoal : public Goal; even when Goal is a complete type. This is still a static cast. The purpose of exporting it is to define it in - a place where `SubstitutionGoal` is concrete, and use it in a place where it + a place where `PathSubstitutionGoal` is concrete, and use it in a place where it is opaque. */ -GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -72,7 +72,7 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ std::map> derivationGoals; - std::map> substitutionGoals; + std::map> substitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -146,7 +146,7 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 03bb0218d..fc67f215a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,7 +281,9 @@ private: void createUser(const std::string & userName, uid_t userId) override; friend struct LocalDerivationGoal; + friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; + friend struct DerivationGoal; }; From df9d4f88d5aed0aa4ed67eb012e9f260550b7200 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 15:40:10 +0100 Subject: [PATCH 171/725] Allow substituting drv outputs when building --- src/libstore/build/derivation-goal.cc | 25 +++-- .../build/drv-output-substitution-goal.cc | 95 +++++++++++++++++++ .../build/drv-output-substitution-goal.hh | 50 ++++++++++ src/libstore/build/worker.cc | 22 ++++- src/libstore/build/worker.hh | 5 + 5 files changed, 185 insertions(+), 12 deletions(-) create mode 100644 src/libstore/build/drv-output-substitution-goal.cc create mode 100644 src/libstore/build/drv-output-substitution-goal.hh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7b97e575a..7dcd2a6eb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -246,17 +246,22 @@ void DerivationGoal::haveDerivation() through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { + for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(upcast_goal(worker.makePathSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + if (!status.known) + addWaitee( + upcast_goal( + worker.makeDrvOutputSubstitutionGoal( + DrvOutput{status.outputHash, outputName}, + buildMode == bmRepair ? Repair : NoRepair + ) + ) + ); + else + addWaitee(upcast_goal(worker.makePathSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv)))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc new file mode 100644 index 000000000..a5ac4c49d --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -0,0 +1,95 @@ +#include "drv-output-substitution-goal.hh" +#include "worker.hh" +#include "substitution-goal.hh" + +namespace nix { + +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional ca) + : Goal(worker) + , id(id) +{ + state = &DrvOutputSubstitutionGoal::init; + name = fmt("substitution of '%s'", id.to_string()); + trace("created"); +} + + +void DrvOutputSubstitutionGoal::init() +{ + trace("init"); + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + tryNext(); +} + +void DrvOutputSubstitutionGoal::tryNext() +{ + trace("Trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + return; + } + + auto sub = subs.front(); + subs.pop_front(); + + // FIXME: Make async + outputInfo = sub->queryRealisation(id); + if (!outputInfo) { + tryNext(); + return; + } + + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); + + if (waitees.empty()) outPathValid(); + else state = &DrvOutputSubstitutionGoal::outPathValid; +} + +void DrvOutputSubstitutionGoal::outPathValid() +{ + assert(outputInfo); + trace("Output path substituted"); + + if (nrFailed > 0) { + debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + worker.store.registerDrvOutput(*outputInfo); + finished(); +} + +void DrvOutputSubstitutionGoal::finished() +{ + trace("finished"); + amDone(ecSuccess); +} + +string DrvOutputSubstitutionGoal::key() +{ + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + std::string(id.to_string()); +} + +void DrvOutputSubstitutionGoal::work() +{ + (this->*state)(); +} + +} diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh new file mode 100644 index 000000000..63ab53d89 --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -0,0 +1,50 @@ +#pragma once + +#include "store-api.hh" +#include "goal.hh" +#include "realisation.hh" + +namespace nix { + +class Worker; + +// Substitution of a derivation output. +// This is done in three steps: +// 1. Fetch the output info from a substituter +// 2. Substitute the corresponding output path +// 3. Register the output info +class DrvOutputSubstitutionGoal : public Goal { +private: + // The drv output we're trying to substitue + DrvOutput id; + + // The realisation corresponding to the given output id. + // Will be filled once we can get it. + std::optional outputInfo; + + /* The remaining substituters. */ + std::list> subs; + + /* Whether a substituter failed. */ + bool substituterFailed = false; + +public: + DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + + typedef void (DrvOutputSubstitutionGoal::*GoalState)(); + GoalState state; + + void init(); + void tryNext(); + void outPathValid(); + void finished(); + + void timedOut(Error && ex) override { abort(); }; + + string key() override; + + void work() override; + +}; + +} diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 619b1d69c..616b17e61 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -90,8 +91,20 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } -template -static void removeGoal(std::shared_ptr goal, std::map> & goalMap) +std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +{ + std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; + auto goal = goal_weak.lock(); // FIXME + if (!goal) { + goal = std::make_shared(id, *this, repair, ca); + goal_weak = goal; + wakeUp(goal); + } + return goal; +} + +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ for (auto i = goalMap.begin(); @@ -111,6 +124,8 @@ void Worker::removeGoal(GoalPtr goal) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, drvOutputSubstitutionGoals); else assert(false); if (topGoals.find(goal) != topGoals.end()) { @@ -474,5 +489,8 @@ void Worker::markContentsGood(const StorePath & path) GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) { + return subGoal; +} } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 42acf8542..918de35f6 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "lock.hh" #include "store-api.hh" #include "goal.hh" +#include "realisation.hh" #include #include @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; struct PathSubstitutionGoal; +class DrvOutputSubstitutionGoal; /* Workaround for not being able to declare a something like @@ -24,6 +26,7 @@ struct PathSubstitutionGoal; a place where `PathSubstitutionGoal` is concrete, and use it in a place where it is opaque. */ GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -73,6 +76,7 @@ private: same derivation / path. */ std::map> derivationGoals; std::map> substitutionGoals; + std::map> drvOutputSubstitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -147,6 +151,7 @@ public: /* substitution goal */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); From 93b5a59b674c0a29846828c7d14b434cc954f8ee Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 9 Nov 2020 16:04:43 +0100 Subject: [PATCH 172/725] Add a test for the remote caching of CA derivations --- tests/ca/substitute.sh | 21 +++++++++++++++++++++ tests/local.mk | 3 ++- tests/push-to-store.sh | 6 ++++-- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 tests/ca/substitute.sh diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh new file mode 100644 index 000000000..79a6ef8b1 --- /dev/null +++ b/tests/ca/substitute.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Ensure that binary substitution works properly with ca derivations + +source common.sh + +sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf + +export REMOTE_STORE=file://$TEST_ROOT/binary_cache + +buildDrvs () { + nix build --file ./content-addressed.nix -L --no-link "$@" +} + +# Populate the remote cache +buildDrvs --post-build-hook ../push-to-store.sh + +# Restart the build on an empty store, ensuring that we don't build +clearStore +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 + diff --git a/tests/local.mk b/tests/local.mk index 07cfd7a50..e17555051 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -41,7 +41,8 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/nix-copy.sh + ca/nix-copy.sh \ + ca/substitute.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh index 6aadb916b..25352c751 100755 --- a/tests/push-to-store.sh +++ b/tests/push-to-store.sh @@ -1,4 +1,6 @@ #!/bin/sh -echo Pushing "$@" to "$REMOTE_STORE" -printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs +set -x + +echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" +printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs From 9931f18c2dfff2642dea8e1a153eaaa58d7e3c8a Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sun, 21 Feb 2021 11:08:28 -0500 Subject: [PATCH 173/725] Add support for bare git repositories with git+file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local git repositories are normally used directly instead of cloning. This commit checks if a repo is bare and forces a clone. Co-authored-by: Théophane Hufschmitt --- src/libfetchers/git.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 81c647f89..4f9db1bcd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -153,12 +153,14 @@ struct GitInputScheme : InputScheme std::pair getActualUrl(const Input & input) const { - // Don't clone file:// URIs (but otherwise treat them the - // same as remote URIs, i.e. don't use the working tree or - // HEAD). + // file:// URIs are normally not cloned (but otherwise treated the + // same as remote URIs, i.e. we don't use the working tree or + // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git + // repo, treat as a remote URI to force a clone. static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); - bool isLocal = url.scheme == "file" && !forceHttp; + bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); + bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; return {isLocal, isLocal ? url.path : url.base}; } From 92a234322f5a46b65825c748220cef40209eeacd Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sun, 21 Feb 2021 10:41:46 -0500 Subject: [PATCH 174/725] Add test for git+file with bare repository --- tests/flakes.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 25ba2ac43..9747aba7a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -25,6 +25,7 @@ templatesDir=$TEST_ROOT/templates nonFlakeDir=$TEST_ROOT/nonFlake flakeA=$TEST_ROOT/flakeA flakeB=$TEST_ROOT/flakeB +flakeGitBare=$TEST_ROOT/flakeGitBare for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do rm -rf $repo $repo.tmp @@ -604,6 +605,11 @@ nix flake update $flake3Dir [[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]] [[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]] +# Test git+file with bare repo. +rm -rf $flakeGitBare +git clone --bare $flake1Dir $flakeGitBare +nix build -o $TEST_ROOT/result git+file://$flakeGitBare + # Test Mercurial flakes. rm -rf $flake5Dir hg init $flake5Dir From 7ce10924c74e9e037b05558aeb5f0639df5955f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Mar 2021 15:07:09 +0000 Subject: [PATCH 175/725] Fix bad wanted output error as requested - UsageError -> Error - include drv path too --- src/libstore/build/derivation-goal.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 530f8829a..4c3bccf25 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1274,7 +1274,9 @@ void DerivationGoal::checkPathValidity() // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) - throw UsageError("some wanted outputs are not provided by the derivation: %s", concatStringsSep(", ", wantedOutputsLeft)); + throw Error("derivation '%s' does not have wanted outputs %s", + worker.store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); } From fc6bfb261d50102016ed812ecf9949d41fe539f7 Mon Sep 17 00:00:00 2001 From: dramforever Date: Tue, 2 Mar 2021 21:56:50 +0800 Subject: [PATCH 176/725] libfetchers/tarball: Lock on effectiveUrl Basically, if a tarball URL is used as a flake input, and the URL leads to a redirect, the final redirect destination would be recorded as the locked URL. This allows tarballs under https://nixos.org/channels to be used as flake inputs. If we, as before, lock on to the original URL it would break every time the channel updates. --- src/libfetchers/fetchers.hh | 8 +++++++- src/libfetchers/github.cc | 6 +++--- src/libfetchers/tarball.cc | 19 ++++++++++++++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a72cfafa4..c6b219c02 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -145,7 +145,13 @@ DownloadFileResult downloadFile( bool immutable, const Headers & headers = {}); -std::pair downloadTarball( +struct DownloadTarballMeta +{ + time_t lastModified; + std::string effectiveUrl; +}; + +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8352ef02d..3e5ad75a8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); + input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", uint64_t(lastModified)} + {"lastModified", uint64_t(meta.lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b8d7d2c70..bd05bb2f1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -109,7 +109,7 @@ DownloadFileResult downloadFile( }; } -std::pair downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -127,7 +127,10 @@ std::pair downloadTarball( if (cached && !cached->expired) return { Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), - getIntAttr(cached->infoAttrs, "lastModified") + { + .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")), + .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url), + }, }; auto res = downloadFile(store, url, name, immutable, headers); @@ -152,6 +155,7 @@ std::pair downloadTarball( Attrs infoAttrs({ {"lastModified", uint64_t(lastModified)}, + {"effectiveUrl", res.effectiveUrl}, {"etag", res.etag}, }); @@ -164,7 +168,10 @@ std::pair downloadTarball( return { Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), - lastModified, + { + .lastModified = lastModified, + .effectiveUrl = res.effectiveUrl, + }, }; } @@ -223,9 +230,11 @@ struct TarballInputScheme : InputScheme return true; } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & _input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + Input input(_input); + auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false); + input.attrs.insert_or_assign("url", meta.effectiveUrl); return {std::move(tree), input}; } }; From 7331da99abead2b59efcfdaf729cb1034642b630 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Feb 2021 13:35:31 +0100 Subject: [PATCH 177/725] Make NIX_SHOW_STATS work with new-style commands --- src/libcmd/command.hh | 2 ++ src/libcmd/installables.cc | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index c02193924..e66c697eb 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -48,6 +48,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs ref getEvalState(); std::shared_ptr evalState; + + ~EvalCommand(); }; struct MixFlakeOptions : virtual Args, EvalCommand diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4739dc974..7102f5a1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -280,6 +280,12 @@ ref EvalCommand::getEvalState() return ref(evalState); } +EvalCommand::~EvalCommand() +{ + if (evalState) + evalState->printStats(); +} + void completeFlakeRef(ref store, std::string_view prefix) { if (prefix == "") From 665d4ec2dac6734caff9de5b030be123cb7276ef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Mar 2021 17:52:57 +0100 Subject: [PATCH 178/725] nix repl :doc: Don't return docs for partially applied primops This gives misleading results for Nixpkgs functions like lib.toUpper. Fixes #4596. --- src/libexpr/eval.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e2f2308aa..3afe2e47b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -592,10 +592,8 @@ Value & EvalState::getBuiltin(const string & name) std::optional EvalState::getDoc(Value & v) { - if (v.isPrimOp() || v.isPrimOpApp()) { + if (v.isPrimOp()) { auto v2 = &v; - while (v2->isPrimOpApp()) - v2 = v2->primOpApp.left; if (v2->primOp->doc) return Doc { .pos = noPos, From e16431b4665c0362f66bace7734fed0a6c0692d5 Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 4 Mar 2021 16:14:23 +0700 Subject: [PATCH 179/725] improve man page for nix.conf (builders) --- src/libstore/globals.hh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a51d9c2f1..bf0767dfa 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -206,7 +206,17 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of `nix.machines`."}; + R"( + A semicolon-separated list of build machines, where each machine follows this format: + + {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}] + + Examples: + + ssh://root@builder1.com + + ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm + )"}; Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", From 6212e89bf604d61fc896f21f66908be6fbbfcc5d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 5 Mar 2021 00:49:46 +0000 Subject: [PATCH 180/725] Avoid some StorePath -> Path -> StorePath roundtrips There were done when StorePath was defined in Rust and there were some FFI issues. This is no longer an issue. --- src/libstore/misc.cc | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..f58816ad8 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -22,55 +22,53 @@ void Store::computeFSClosure(const StorePathSet & startPaths, Sync state_(State{0, paths_, 0}); - std::function enqueue; + std::function enqueue; std::condition_variable done; - enqueue = [&](const Path & path) -> void { + enqueue = [&](const StorePath & path) -> void { { auto state(state_.lock()); if (state->exc) return; - if (!state->paths.insert(parseStorePath(path)).second) return; + if (!state->paths.insert(path).second) return; state->pending++; } - queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future> fut) { + queryPathInfo(path, {[&](std::future> fut) { // FIXME: calls to isValidPath() should be async try { auto info = fut.get(); - auto path = parseStorePath(pathS); - if (flipDirection) { StorePathSet referrers; queryReferrers(path, referrers); for (auto & ref : referrers) if (ref != path) - enqueue(printStorePath(ref)); + enqueue(ref); if (includeOutputs) for (auto & i : queryValidDerivers(path)) - enqueue(printStorePath(i)); + enqueue(i); if (includeDerivers && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(printStorePath(i)); + enqueue(i); } else { for (auto & ref : info->references) if (ref != path) - enqueue(printStorePath(ref)); + enqueue(ref); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(printStorePath(i)); + if (isValidPath(i)) enqueue(i); if (includeDerivers && info->deriver && isValidPath(*info->deriver)) - enqueue(printStorePath(*info->deriver)); + enqueue(*info->deriver); } @@ -90,7 +88,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, }; for (auto & startPath : startPaths) - enqueue(printStorePath(startPath)); + enqueue(startPath); { auto state(state_.lock()); @@ -160,13 +158,10 @@ void Store::queryMissing(const std::vector & targets, }; auto checkOutput = [&]( - const Path & drvPathS, ref drv, const Path & outPathS, ref> drvState_) + const StorePath & drvPath, ref drv, const StorePath & outPath, ref> drvState_) { if (drvState_->lock()->done) return; - auto drvPath = parseStorePath(drvPathS); - auto outPath = parseStorePath(outPathS); - SubstitutablePathInfos infos; querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); @@ -203,7 +198,7 @@ void Store::queryMissing(const std::vector & targets, return; } - PathSet invalid; + StorePathSet invalid; /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; @@ -213,7 +208,7 @@ void Store::queryMissing(const std::vector & targets, break; } if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) - invalid.insert(printStorePath(*pathOpt)); + invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; @@ -223,7 +218,7 @@ void Store::queryMissing(const std::vector & targets, if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); } else mustBuildDrv(path.path, *drv); From 6e849e3b0a6eb46e6dc65cbd091cc829eab09a5f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 3 Mar 2021 14:46:15 -0800 Subject: [PATCH 181/725] nix-build: set execfail When starting a nix-shell with `-i` it was previously possible for it to silently fail in the scenario where the specified interpreter didn't exist. This happened due to the `exec` call masking the issue. With this change we enable `execfail`, which causes the script using `nix-shell` as interpreter to correctly exit with code 127. Fixes: #4598 --- src/nix-build/nix-build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7b4a53919..65b85b304 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -447,6 +447,7 @@ static void main_nix_build(int argc, char * * argv) "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " "unset TZ; %6%" + "shopt -s execfail;" "%7%", shellEscape(tmpDir), (pure ? "" : "p=$PATH; "), From ac8ba2eae4fc649d7a3a19815631b4d76e60d74a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sat, 6 Mar 2021 19:51:29 -0600 Subject: [PATCH 182/725] remove doc for obsolete --no-build-hook flag `--no-build-hook` appears to have been removed in 25f32625e2f2a3a1e1b3a3811da82f21c3a3b880 --- doc/manual/src/command-ref/opt-common.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 9650f53f8..bc8eb6796 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -134,15 +134,6 @@ Most Nix commands accept the following command-line options: failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). - - `--no-build-hook` - Disables the build hook mechanism. This allows to ignore remote - builders if they are setup on the machine. - - It's useful in cases where the bandwidth between the client and the - remote builder is too low. In that case it can take more time to - upload the sources to the remote builder and fetch back the result - than to do the computation locally. - - `--readonly-mode` When this option is used, no attempt is made to open the Nix database. Most Nix operations do need database access, so those From 89013bdd7ed4007871cc421315b51b7cada8edff Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 9 Mar 2021 10:11:25 +0100 Subject: [PATCH 183/725] Add a `nix realisation` command for working on realisations Currently only has `nix realisation info`, more to come probably --- src/nix/realisation.cc | 78 +++++++++++++++++++++++++++++++++++++ src/nix/realisation/info.md | 15 +++++++ 2 files changed, 93 insertions(+) create mode 100644 src/nix/realisation.cc create mode 100644 src/nix/realisation/info.md diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc new file mode 100644 index 000000000..9ee9ccb91 --- /dev/null +++ b/src/nix/realisation.cc @@ -0,0 +1,78 @@ +#include "command.hh" +#include "common-args.hh" + +#include + +using namespace nix; + +struct CmdRealisation : virtual NixMultiCommand +{ + CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + { } + + std::string description() override + { + return "manipulate a Nix realisation"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix realisation' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRealisation = registerCommand("realisation"); + +struct CmdRealisationInfo : RealisedPathsCommand, MixJSON +{ + std::string description() override + { + return "query information about one or several realisations"; + } + + std::string doc() override + { + return + #include "realisation/info.md" + ; + } + + Category category() override { return catSecondary; } + + void run(ref store, std::vector paths) override + { + settings.requireExperimentalFeature("ca-derivations"); + if (json) { + nlohmann::json res = nlohmann::json::array(); + for (auto & path : paths) { + nlohmann::json currentPath; + if (auto realisation = std::get_if(&path.raw)) + currentPath = realisation->toJSON(); + else + currentPath["opaquePath"] = store->printStorePath(path.path()); + + res.push_back(currentPath); + } + std::cout << res.dump(); + } + else { + for (auto & path : paths) { + if (auto realisation = std::get_if(&path.raw)) { + std::cout << + realisation->id.to_string() << " " << + store->printStorePath(realisation->outPath); + } else + std::cout << store->printStorePath(path.path()); + + std::cout << std::endl; + } + } + } +}; + +static auto rCmdRealisationInfo = registerCommand2({"realisation", "info"}); diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md new file mode 100644 index 000000000..852240f44 --- /dev/null +++ b/src/nix/realisation/info.md @@ -0,0 +1,15 @@ +R"MdBoundary( +# Description + +Display some informations about the given realisation + +# Examples + +Show some information about the realisation of the `hello` package: + +```console +$ nix realisation info nixpkgs#hello --json +[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}] +``` + +)MdBoundary" From 8a0c00b85600991cdb9aa05902defec6ac44b777 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 10 Dec 2019 15:47:38 +0700 Subject: [PATCH 184/725] Use libarchive for all compression --- src/libstore/filetransfer.cc | 2 +- src/libutil/compression.cc | 420 +++++++++-------------------------- src/libutil/compression.hh | 10 +- src/libutil/serialise.cc | 56 ++++- src/libutil/serialise.hh | 8 + src/libutil/tarfile.cc | 104 ++++----- src/libutil/tarfile.hh | 19 ++ 7 files changed, 241 insertions(+), 378 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8ea5cdc9d..514ab3bf9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -148,7 +148,7 @@ struct curlFileTransfer : public FileTransfer } LambdaSink finalSink; - std::shared_ptr decompressionSink; + std::shared_ptr decompressionSink; std::optional errorSink; std::exception_ptr writeException; diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 986ba2976..8ba536000 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,10 +1,13 @@ #include "compression.hh" +#include "tarfile.hh" #include "util.hh" #include "finally.hh" #include "logging.hh" #include #include +#include +#include #include #include @@ -35,6 +38,80 @@ struct ChunkedCompressionSink : CompressionSink virtual void writeInternal(std::string_view data) = 0; }; +struct ArchiveDecompressionSource : Source +{ + std::unique_ptr archive = 0; + Source & src; + ArchiveDecompressionSource(Source & src) : src(src) {} + ~ArchiveDecompressionSource() override {} + size_t read(char * data, size_t len) override { + struct archive_entry* ae; + if (!archive) { + archive = std::make_unique(src, true); + this->archive->check(archive_read_next_header(this->archive->archive, &ae), "Failed to read header (%s)"); + if (archive_filter_count(this->archive->archive) < 2) { + throw CompressionError("Input compression not recognized."); + } + } + ssize_t result = archive_read_data(this->archive->archive, data, len); + if (result > 0) return result; + if (result == 0) { + throw EndOfFile("reached end of compressed file"); + } + this->archive->check(result, "Failed to read compressed data (%s)"); + return result; + } +}; +struct ArchiveCompressionSink : CompressionSink +{ + Sink & nextSink; + struct archive* archive; + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { + archive = archive_write_new(); + if (!archive) throw Error("failed to initialize libarchive"); + check(archive_write_add_filter_by_name(archive, format.c_str()), "Couldn't initialize compression (%s)"); + check(archive_write_set_format_raw(archive)); + if (format == "xz" && parallel) { + check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); + } + // disable internal buffering + check(archive_write_set_bytes_per_block(archive, 0)); + // disable output padding + check(archive_write_set_bytes_in_last_block(archive, 1)); + open(); + } + ~ArchiveCompressionSink() override { + if (archive) archive_write_free(archive); + } + void finish() override { + flush(); + check(archive_write_close(archive)); + } + void check(int err, const char *reason="Failed to compress (%s)") { + if (err == ARCHIVE_EOF) + throw EndOfFile("reached end of archive"); + else if (err != ARCHIVE_OK) + throw Error(reason, archive_error_string(this->archive)); + } + void write(std::string_view data) override { + ssize_t result = archive_write_data(archive, data.data(), data.length()); + if (result <= 0) check(result); + } +private: + void open() { + check(archive_write_open(archive, this, NULL, ArchiveCompressionSink::callback_write, NULL)); + struct archive_entry *ae = archive_entry_new(); + archive_entry_set_filetype(ae, AE_IFREG); + check(archive_write_header(archive, ae)); + archive_entry_free(ae); + } + static ssize_t callback_write(struct archive *archive, void *_self, const void *buffer, size_t length) { + ArchiveCompressionSink *self = (ArchiveCompressionSink *)_self; + self->nextSink({(const char*)buffer, length}); + return length; + } +}; + struct NoneSink : CompressionSink { Sink & nextSink; @@ -43,171 +120,6 @@ struct NoneSink : CompressionSink void write(std::string_view data) override { nextSink(data); } }; -struct GzipDecompressionSink : CompressionSink -{ - Sink & nextSink; - z_stream strm; - bool finished = false; - uint8_t outbuf[BUFSIZ]; - - GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - - // Enable gzip and zlib decoding (+32) with 15 windowBits - int ret = inflateInit2(&strm,15+32); - if (ret != Z_OK) - throw CompressionError("unable to initialise gzip encoder"); - } - - ~GzipDecompressionSink() - { - inflateEnd(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (Bytef *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - int ret = inflate(&strm,Z_SYNC_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) - throw CompressionError("error while decompressing gzip file: %d (%d, %d)", - zError(ret), data.size(), strm.avail_in); - - finished = ret == Z_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (Bytef *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct XzDecompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; - - XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - lzma_ret ret = lzma_stream_decoder( - &strm, UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma decoder"); - - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~XzDecompressionSink() - { - lzma_end(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - strm.next_in = (const unsigned char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while decompressing xz file", ret); - - finished = ret == LZMA_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct BzipDecompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; - - BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 decoder"); - - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~BzipDecompressionSink() - { - BZ2_bzDecompressEnd(&strm); - } - - void finish() override - { - flush(); - write({}); - } - - void writeInternal(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (char *) data.data(); - strm.avail_in = data.size(); - - while (strm.avail_in) { - checkInterrupt(); - - int ret = BZ2_bzDecompress(&strm); - if (ret != BZ_OK && ret != BZ_STREAM_END) - throw CompressionError("error while decompressing bzip2 file"); - - finished = ret == BZ_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - struct BrotliDecompressionSink : ChunkedCompressionSink { Sink & nextSink; @@ -261,161 +173,32 @@ struct BrotliDecompressionSink : ChunkedCompressionSink ref decompress(const std::string & method, const std::string & in) { - StringSink ssink; - auto sink = makeDecompressionSink(method, ssink); - (*sink)(in); - sink->finish(); - return ssink.s; + if (method == "br") { + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; + } else { + StringSource ssrc(in); + auto src = makeDecompressionSource(ssrc); + return make_ref(src->drain()); + } } -ref makeDecompressionSink(const std::string & method, Sink & nextSink) +std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink) { if (method == "none" || method == "") - return make_ref(nextSink); - else if (method == "xz") - return make_ref(nextSink); - else if (method == "bzip2") - return make_ref(nextSink); - else if (method == "gzip") - return make_ref(nextSink); + return std::make_unique(nextSink); else if (method == "br") - return make_ref(nextSink); + return std::make_unique(nextSink); else - throw UnknownCompressionMethod("unknown compression method '%s'", method); + return sourceToSink([&](Source & source) { + auto decompressionSource = makeDecompressionSource(source); + decompressionSource->drainInto(nextSink); + }); } -struct XzCompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; - - XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - bool done = false; - - if (parallel) { -#ifdef HAVE_LZMA_MT - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt(&strm, &mt_options); - done = true; -#else - printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); -#endif - } - - if (!done) - ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); - - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma encoder"); - - // FIXME: apply the x86 BCJ filter? - - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~XzCompressionSink() - { - lzma_end(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - strm.next_in = (const unsigned char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while compressing xz file", ret); - - finished = ret == LZMA_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct BzipCompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; - - BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 encoder"); - - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~BzipCompressionSink() - { - BZ2_bzCompressEnd(&strm); - } - - void finish() override - { - flush(); - writeInternal({}); - } - - void writeInternal(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH); - if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) - throw CompressionError("error %d while compressing bzip2 file", ret); - - finished = ret == BZ_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - struct BrotliCompressionSink : ChunkedCompressionSink { Sink & nextSink; @@ -468,15 +251,20 @@ struct BrotliCompressionSink : ChunkedCompressionSink } } }; +std::unique_ptr makeDecompressionSource(Source & prev) { + return std::unique_ptr(new ArchiveDecompressionSource(prev)); +} ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + std::vector la_supports = { + "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" + }; + if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { + return make_ref(nextSink, method, parallel); + } if (method == "none") return make_ref(nextSink); - else if (method == "xz") - return make_ref(nextSink, parallel); - else if (method == "bzip2") - return make_ref(nextSink); else if (method == "br") return make_ref(nextSink); else diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index dd666a4e1..192cb3e91 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,14 +8,18 @@ namespace nix { -struct CompressionSink : BufferedSink +struct CompressionSink : BufferedSink, FinishSink { - virtual void finish() = 0; + using BufferedSink::operator (); + using BufferedSink::write; + using FinishSink::finish; }; +std::unique_ptr makeDecompressionSource(Source & prev); + ref decompress(const std::string & method, const std::string & in); -ref makeDecompressionSink(const std::string & method, Sink & nextSink); +std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink); ref compress(const std::string & method, const std::string & in, const bool parallel = false); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d1a16b6ba..374b48d79 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -201,6 +201,61 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::unique_ptr sourceToSink(std::function fun) +{ + struct SourceToSink : FinishSink + { + typedef boost::coroutines2::coroutine coro_t; + + std::function fun; + std::optional coro; + + SourceToSink(std::function fun) : fun(fun) + { + } + + std::string_view cur; + + void operator () (std::string_view in) override + { + if (in.empty()) return; + cur = in; + + if (!coro) + coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { + LambdaSource source([&](char *out, size_t out_len) { + if (cur.empty()) { + yield(); + if (yield.get()) { + return (size_t)0; + } + } + + size_t n = std::min(cur.size(), out_len); + memcpy(out, cur.data(), n); + cur.remove_prefix(n); + return n; + }); + fun(source); + }); + + if (!*coro) { abort(); } + + if (!cur.empty()) (*coro)(false); + } + + void finish() { + if (!coro) return; + if (!*coro) abort(); + (*coro)(true); + if (*coro) abort(); + } + }; + + return std::make_unique(fun); +} + + std::unique_ptr sinkToSource( std::function fun, std::function eof) @@ -212,7 +267,6 @@ std::unique_ptr sinkToSource( std::function fun; std::function eof; std::optional coro; - bool started = false; SinkToSource(std::function fun, std::function eof) : fun(fun), eof(eof) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5bbbc7ce3..0fe6e8332 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -25,6 +25,13 @@ struct NullSink : Sink { } }; + +struct FinishSink : virtual Sink +{ + virtual void finish() = 0; +}; + + /* A buffered abstract sink. Warning: a BufferedSink should not be used from multiple threads concurrently. */ struct BufferedSink : virtual Sink @@ -281,6 +288,7 @@ struct ChainSource : Source size_t read(char * data, size_t len) override; }; +std::unique_ptr sourceToSink(std::function fun); /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 2da169ba7..b5e1cb4c0 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,83 +2,73 @@ #include #include "serialise.hh" +#include "tarfile.hh" namespace nix { +static int callback_open(struct archive *, void *self) { + return ARCHIVE_OK; +} -struct TarArchive { - struct archive * archive; - Source * source; - std::vector buffer; +static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) { + TarArchive *self = (TarArchive *)_self; + *buffer = self->buffer.data(); - void check(int err, const char * reason = "failed to extract archive: %s") - { + try { + return self->source->read((char *) self->buffer.data(), 4096); + } catch (EndOfFile &) { + return 0; + } catch (std::exception &err) { + archive_set_error(archive, EIO, "Source threw exception: %s", err.what()); + + return -1; + } +} + +static int callback_close(struct archive *, void *self) { + return ARCHIVE_OK; +} + +void TarArchive::check(int err, const char *reason) +{ if (err == ARCHIVE_EOF) throw EndOfFile("reached end of archive"); else if (err != ARCHIVE_OK) throw Error(reason, archive_error_string(this->archive)); } - TarArchive(Source & source) : buffer(4096) - { - this->archive = archive_read_new(); - this->source = &source; +TarArchive::TarArchive(Source& source, bool raw) : buffer(4096) +{ + this->archive = archive_read_new(); + this->source = &source; + if (!raw) { archive_read_support_filter_all(archive); archive_read_support_format_all(archive); - check(archive_read_open(archive, - (void *)this, - TarArchive::callback_open, - TarArchive::callback_read, - TarArchive::callback_close), - "failed to open archive: %s"); - } - - TarArchive(const Path & path) - { - this->archive = archive_read_new(); - + } else { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); - check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); + archive_read_support_format_raw(archive); + archive_read_support_format_empty(archive); } + check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); +} - TarArchive(const TarArchive &) = delete; - void close() - { - check(archive_read_close(archive), "failed to close archive: %s"); - } +TarArchive::TarArchive(const Path &path) +{ + this->archive = archive_read_new(); - ~TarArchive() - { - if (this->archive) archive_read_free(this->archive); - } + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); +} -private: +void TarArchive::close() { + check(archive_read_close(this->archive), "Failed to close archive (%s)"); +} - static int callback_open(struct archive *, void * self) { - return ARCHIVE_OK; - } - - static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) - { - auto self = (TarArchive *)_self; - *buffer = self->buffer.data(); - - try { - return self->source->read((char *) self->buffer.data(), 4096); - } catch (EndOfFile &) { - return 0; - } catch (std::exception & err) { - archive_set_error(archive, EIO, "source threw exception: %s", err.what()); - return -1; - } - } - - static int callback_close(struct archive *, void * self) { - return ARCHIVE_OK; - } -}; +TarArchive::~TarArchive() { + if (this->archive) archive_read_free(this->archive); +} static void extract_archive(TarArchive & archive, const Path & destDir) { diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 89a024f1d..18adf3490 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,7 +1,26 @@ #include "serialise.hh" +#include namespace nix { +struct TarArchive { + struct archive *archive; + Source *source; + std::vector buffer; + + void check(int err, const char *reason = "Failed to extract archive (%s)"); + + TarArchive(Source& source, bool raw = false); + + TarArchive(const Path &path); + + // disable copy constructor + TarArchive(const TarArchive&) = delete; + + void close(); + + ~TarArchive(); +}; void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); From 0431cf6d0992e7986afbb3d0ffd0a7e1cca8ae8a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sun, 7 Feb 2021 15:34:24 -0600 Subject: [PATCH 185/725] fix nixbld user name/uid for macOS --- scripts/bigsur-nixbld-user-migration.sh | 46 +++++++++++++++++++++++++ scripts/install-darwin-multi-user.sh | 2 ++ scripts/install-multi-user.sh | 6 ++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100755 scripts/bigsur-nixbld-user-migration.sh diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh new file mode 100755 index 000000000..f1619fd56 --- /dev/null +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +((NEW_NIX_FIRST_BUILD_UID=301)) + +id_available(){ + dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +} + +change_nixbld_names_and_ids(){ + local name uid next_id + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + echo "Attempting to migrate nixbld users." + echo "Each user should change from nixbld# to _nixbld#" + echo "and their IDs relocated to $next_id+" + while read -r name uid; do + echo " Checking $name (uid: $uid)" + # iterate for a clean ID + while id_available "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." + exit 1 + fi + done + + if [[ $name == _* ]]; then + echo " It looks like $name has already been renamed--skipping." + else + # first 3 are cleanup, it's OK if they aren't here + sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + # remove existing user from group + sudo dseditgroup -o edit -t user -d $name nixbld || true + sudo dscl . change /Users/$name UniqueID $uid $next_id + sudo dscl . change /Users/$name RecordName $name _$name + # add renamed user to group + sudo dseditgroup -o edit -t user -a _$name nixbld + echo " $name migrated to _$name (uid: $next_id)" + fi + done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) +} + +change_nixbld_names_and_ids diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index a27be2a43..f6575ae2f 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -4,6 +4,8 @@ set -eu set -o pipefail readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +NIX_FIRST_BUILD_UID="301" +NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" dsclattr() { /usr/bin/dscl . -read "$1" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5e8b4ac18..30ccf1764 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,7 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" +# darwin installer needs to override these +NIX_FIRST_BUILD_UID="30001" +NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -104,7 +106,7 @@ EOF } nix_user_for_core() { - printf "nixbld%d" "$1" + printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1" } nix_uid_for_core() { From 826877cabf9374e0acd5408c6975ee332b1cccc8 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 11:56:33 +0100 Subject: [PATCH 186/725] Add some logic for signing realisations Not exposed anywhere, but built realisations are now signed (and this should be forwarded when copy-ing them around) --- src/libstore/build/local-derivation-goal.cc | 12 ++++-- src/libstore/ca-specific-schema.sql | 1 + src/libstore/local-store.cc | 29 ++++++++++--- src/libstore/local-store.hh | 4 +- src/libstore/realisation.cc | 46 ++++++++++++++++++++- src/libstore/realisation.hh | 8 ++++ src/libstore/store-api.hh | 5 +++ 7 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c2f1dda6..048135ccf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2615,10 +2615,14 @@ void LocalDerivationGoal::registerOutputs() but it's fine to do in all cases. */ if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) - worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, - .outPath = newInfo.path}); + for (auto& [outputName, newInfo] : infos) { + auto thisRealisation = Realisation{ + .id = DrvOutput{initialOutputs.at(outputName).outputHash, + outputName}, + .outPath = newInfo.path}; + getLocalStore().signRealisation(thisRealisation); + worker.store.registerDrvOutput(thisRealisation); + } } } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 93c442826..20ee046a1 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -6,6 +6,7 @@ create table if not exists Realisations ( drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, + signatures text, -- space-separated list primary key (drvPath, outputName), foreign key (outputPath) references ValidPaths(id) on delete cascade ); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 90fb4a4bd..6bc963f27 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -310,13 +310,13 @@ LocalStore::LocalStore(const Params & params) if (settings.isExperimentalFeatureEnabled("ca-derivations")) { state->stmts->RegisterRealisedOutput.create(state->db, R"( - insert or replace into Realisations (drvPath, outputName, outputPath) - values (?, ?, (select id from ValidPaths where path = ?)) + insert or replace into Realisations (drvPath, outputName, outputPath, signatures) + values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path from Realisations + select Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -662,6 +662,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) .exec(); }); } @@ -1107,6 +1108,11 @@ bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) return requireSigs && !info.checkSignatures(*this, getPublicKeys()); } +bool LocalStore::realisationIsUntrusted(const Realisation & realisation) +{ + return requireSigs && !realisation.checkSignatures(getPublicKeys()); +} + void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -1612,6 +1618,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si } +void LocalStore::signRealisation(Realisation & realisation) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + realisation.sign(secretKey); + } +} + void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. @@ -1649,8 +1667,9 @@ std::optional LocalStore::queryRealisation( if (!use.next()) return std::nullopt; auto outputPath = parseStorePath(use.getStr(0)); - return Ret{ - Realisation{.id = id, .outPath = outputPath}}; + auto signatures = tokenizeString(use.getStr(1)); + return Ret{Realisation{ + .id = id, .outPath = outputPath, .signatures = signatures}}; }); } } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fc67f215a..d54609f01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -137,6 +137,7 @@ public: SubstitutablePathInfos & infos) override; bool pathInfoIsTrusted(const ValidPathInfo &) override; + bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -272,9 +273,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo using the secret keys + /* Add signatures to a ValidPathInfo or Realisation using the secret keys specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); + void signRealisation(Realisation &); Path getRealStoreDir() override { return realStoreDir; } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index cd74af4ee..638065547 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -25,27 +25,69 @@ nlohmann::json Realisation::toJSON() const { return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, + {"signatures", signatures}, }; } Realisation Realisation::fromJSON( const nlohmann::json& json, const std::string& whence) { - auto getField = [&](std::string fieldName) -> std::string { + auto getOptionalField = [&](std::string fieldName) -> std::optional { auto fieldIterator = json.find(fieldName); if (fieldIterator == json.end()) + return std::nullopt; + return *fieldIterator; + }; + auto getField = [&](std::string fieldName) -> std::string { + if (auto field = getOptionalField(fieldName)) + return *field; + else throw Error( "Drv output info file '%1%' is corrupt, missing field %2%", whence, fieldName); - return *fieldIterator; }; + StringSet signatures; + if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) + signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), + .signatures = signatures, }; } +std::string Realisation::fingerprint() const +{ + auto serialized = toJSON(); + serialized.erase("signatures"); + return serialized.dump(); +} + +void Realisation::sign(const SecretKey & secretKey) +{ + signatures.insert(secretKey.signDetached(fingerprint())); +} + +bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + +size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const +{ + // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to + // an input-addressed one − because in that case the drv is enough to check + // it − but we can't know that here. + + size_t good = 0; + for (auto & sig : signatures) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index fc92d3c17..f5049c9e9 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,7 @@ #include "path.hh" #include #include "comparator.hh" +#include "crypto.hh" namespace nix { @@ -25,9 +26,16 @@ struct Realisation { DrvOutput id; StorePath outPath; + StringSet signatures; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + std::string fingerprint() const; + void sign(const SecretKey &); + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + size_t checkSignatures(const PublicKeys & publicKeys) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 71a28eeb8..0cd56d34e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -389,6 +389,11 @@ public: return true; } + virtual bool realisationIsUntrusted(const Realisation & ) + { + return true; + } + protected: virtual void queryPathInfoUncached(const StorePath & path, From 3e6017f911127555cfbed71fe4a4df8f70d08bbb Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 15:07:33 +0100 Subject: [PATCH 187/725] pathInfoIsTrusted -> pathInfoIsUntrusted I guess the rationale behind the old name wath that `pathInfoIsTrusted(info)` returns `true` iff we would need to `blindly` trust the path (because it has no valid signature and `requireSigs` is set), but I find it to be a really confusing footgun because it's quite natural to give it the opposite meaning. --- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/local-store.hh | 2 +- src/libstore/store-api.hh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 5d88b8758..7b1ac126e 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -142,7 +142,7 @@ void PathSubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("substituter '%s' does not have a valid signature for path '%s'", sub->getUri(), worker.store.printStorePath(storePath)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6bc963f27..950a9f74e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1103,7 +1103,7 @@ const PublicKeys & LocalStore::getPublicKeys() return *state->publicKeys; } -bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { return requireSigs && !info.checkSignatures(*this, getPublicKeys()); } @@ -1116,7 +1116,7 @@ bool LocalStore::realisationIsUntrusted(const Realisation & realisation) void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (checkSigs && pathInfoIsTrusted(info)) + if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d54609f01..c311d295a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -136,7 +136,7 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - bool pathInfoIsTrusted(const ValidPathInfo &) override; + bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0cd56d34e..b90aeaa4c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -384,7 +384,7 @@ public: we don't really want to add the dependencies listed in a nar info we don't trust anyyways. */ - virtual bool pathInfoIsTrusted(const ValidPathInfo &) + virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; } From 54ced9072b94515a756e1e8e76c92a42f0ccf366 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 16:43:11 +0100 Subject: [PATCH 188/725] Check the signatures when copying store paths around Broken atm --- src/libstore/local-store.cc | 8 ++++++++ src/libstore/local-store.hh | 1 + src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 ++ tests/ca/signatures.sh | 39 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 3 ++- 6 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/ca/signatures.sh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 950a9f74e..83daa7506 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -652,6 +652,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } } +void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) +{ + settings.requireExperimentalFeature("ca-derivations"); + if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) + registerDrvOutput(info); + else + throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); +} void LocalStore::registerDrvOutput(const Realisation & info) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c311d295a..26e034a82 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,7 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ void registerDrvOutput(const Realisation & info) override; + void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); std::optional queryRealisation(const DrvOutput&) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77c310988..5e321cedf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -798,7 +798,7 @@ std::map copyPaths(ref srcStore, ref dstStor auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation); + dstStore->registerDrvOutput(realisation, checkSigs); } } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b90aeaa4c..5d19e8949 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -485,6 +485,8 @@ public: */ virtual void registerDrvOutput(const Realisation & output) { unsupported("registerDrvOutput"); } + virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) + { return registerDrvOutput(output); } /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; diff --git a/tests/ca/signatures.sh b/tests/ca/signatures.sh new file mode 100644 index 000000000..4b4e468f7 --- /dev/null +++ b/tests/ca/signatures.sh @@ -0,0 +1,39 @@ +source common.sh + +# Globally enable the ca derivations experimental flag +sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" + +clearStore +clearCache + +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +pk1=$(cat $TEST_ROOT/pk1) + +export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" +export REMOTE_STORE="file://$REMOTE_STORE_DIR" + +ensureCorrectlyCopied () { + attrPath="$1" + nix build --store "$REMOTE_STORE" --file ./content-addressed.nix "$attrPath" +} + +testOneCopy () { + clearStore + rm -rf "$REMOTE_STORE_DIR" + + attrPath="$1" + nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + --secret-key-files "$TEST_ROOT/sk1" + + ensureCorrectlyCopied "$attrPath" + + # Ensure that we can copy back what we put in the store + clearStore + nix copy --from $REMOTE_STORE \ + --file ./content-addressed.nix "$attrPath" \ + --trusted-public-keys $pk1 +} + +for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do + testOneCopy "$attrPath" +done diff --git a/tests/local.mk b/tests/local.mk index e17555051..9a227bec5 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -41,8 +41,9 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/nix-copy.sh \ ca/substitute.sh + ca/signatures.sh \ + ca/nix-copy.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 703c98c6cb922ff9d8cd8cb2c1104e0d3b15b803 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Mar 2021 17:32:20 +0100 Subject: [PATCH 189/725] Properly sign the unresolved drvs Don't let them inherit the signature from the parent one (because it makes no sense to do so), but re-sign them after they have been built --- src/libstore/build/derivation-goal.cc | 2 ++ src/libstore/build/derivation-goal.hh | 3 +++ src/libstore/build/local-derivation-goal.cc | 7 ++++++- src/libstore/build/local-derivation-goal.hh | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7dcd2a6eb..d624e58b9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -925,6 +925,8 @@ void DerivationGoal::resolvedFinished() { if (realisation) { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; + newRealisation.signatures.clear(); + signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index c85bcd84f..704b77caf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -180,6 +180,9 @@ struct DerivationGoal : public Goal /* Open a log file and a pipe to it. */ Path openLogFile(); + /* Sign the newly built realisation if the store allows it */ + virtual void signRealisation(Realisation&) {} + /* Close the log file. */ void closeLogFile(); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 048135ccf..2966bb565 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2620,12 +2620,17 @@ void LocalDerivationGoal::registerOutputs() .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, .outPath = newInfo.path}; - getLocalStore().signRealisation(thisRealisation); + signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } } } +void LocalDerivationGoal::signRealisation(Realisation & realisation) +{ + getLocalStore().signRealisation(realisation); +} + void LocalDerivationGoal::checkOutputs(const std::map & outputs) { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 4bbf27a1b..47b818a8b 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -161,6 +161,8 @@ struct LocalDerivationGoal : public DerivationGoal as valid. */ void registerOutputs() override; + void signRealisation(Realisation &) override; + /* Check that an output meets the requirements specified by the 'outputChecks' attribute (or the legacy '{allowed,disallowed}{References,Requisites}' attributes). */ From 5869b3025d8ed2b99a8dca61f335789ce6dc83e1 Mon Sep 17 00:00:00 2001 From: sternenseemann <0rpkxez4ksa01gb3typccl0i@systemli.org> Date: Tue, 16 Mar 2021 02:42:14 +0100 Subject: [PATCH 190/725] tests/local.mk: fix missing newline escape Fixes syntax error introduced in 54ced9072b94515a756e1e8e76c92a42f0ccf366. --- tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index 1ca363091..de095c117 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -42,7 +42,7 @@ nix_tests = \ build.sh \ compute-levels.sh \ ca/build.sh \ - ca/substitute.sh + ca/substitute.sh \ ca/signatures.sh \ ca/nix-copy.sh # parallel.sh From 5716345adf2e794fd62229ea52352e74e92e8e63 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 10 Nov 2020 10:43:33 +0100 Subject: [PATCH 191/725] Add a test ensuring compatibility with an old daemon This requires adding `nix` to its own closure which is a bit unfortunate, but as it is optional (the test will be disabled if `OUTER_NIX` is unset) it shouldn't be too much of an issue. (Ideally this should go in another derivation so that we can build Nix and run the test independently, but as the tests are running in the same derivation as the build it's a bit complicated to do so). --- flake.nix | 9 +++++++++ tests/common.sh.in | 3 +-- tests/local.mk | 2 +- tests/remote-store-old-daemon.sh | 7 +++++++ tests/remote-store.sh | 4 ++-- 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/remote-store-old-daemon.sh diff --git a/flake.nix b/flake.nix index e59ec9a35..3e236aaca 100644 --- a/flake.nix +++ b/flake.nix @@ -150,6 +150,11 @@ # 'nix.perl-bindings' packages. overlay = final: prev: { + # An older version of Nix to test against when using the daemon. + # Currently using `nixUnstable` as the stable one doesn't respect + # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. + mainstream-nix = prev.nixUnstable; + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -158,6 +163,8 @@ VERSION_SUFFIX = versionSuffix; + OUTER_NIX = mainstream-nix; + outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; @@ -486,6 +493,8 @@ stdenv.mkDerivation { name = "nix"; + OUTER_NIX = mainstream-nix; + outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; diff --git a/tests/common.sh.in b/tests/common.sh.in index de44a4da4..277dd6dfa 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -57,7 +57,6 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" - nix-store --init clearProfiles } @@ -73,7 +72,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_STATE_DIR/daemon-socket/socket - nix daemon & + ${NIX_DAEMON_COMMAND:-nix daemon} & for ((i = 0; i < 30; i++)); do if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 diff --git a/tests/local.mk b/tests/local.mk index de095c117..dd9a0ad56 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - local-store.sh remote-store.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh new file mode 100644 index 000000000..ede7ce716 --- /dev/null +++ b/tests/remote-store-old-daemon.sh @@ -0,0 +1,7 @@ +# Test that the new Nix can properly talk to an old daemon. +# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test + +if [[ -n "$OUTER_NIX" ]]; then + export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon + source remote-store.sh +fi diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f7ae1a2ed..31210ab47 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -23,12 +23,12 @@ startDaemon storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh +nix-store --gc --max-freed 1K + nix-store --dump-db > $TEST_ROOT/d1 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 cmp $TEST_ROOT/d1 $TEST_ROOT/d2 -nix-store --gc --max-freed 1K - killDaemon user=$(whoami) From eab9cdbd75e739be33f9433cfba9ab354d084440 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 17 Nov 2020 14:33:09 +0100 Subject: [PATCH 192/725] Add a test for the migration of the db between versions --- tests/db-migration.sh | 25 +++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 26 insertions(+) create mode 100644 tests/db-migration.sh diff --git a/tests/db-migration.sh b/tests/db-migration.sh new file mode 100644 index 000000000..e6a405770 --- /dev/null +++ b/tests/db-migration.sh @@ -0,0 +1,25 @@ +# Test that we can successfully migrate from an older db schema + +# Only run this if we have an older Nix available +if [[ -z "$OUTER_NIX" ]]; then + exit 0 +fi + +source common.sh + +# Fill the db using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$OUTER_NIX/bin:$PATH" +clearStore +nix-build simple.nix --no-out-link +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +dependenciesOutPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1") +fixedOutPath=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build fixed.nix -A good.0 --no-out-link) + +# Migrate to the new schema and ensure that everything's there +export PATH="$PATH_WITH_NEW_NIX" +info=$(nix path-info --json $dependenciesOutPath) +[[ $info =~ '"ultimate":true' ]] +[[ $info =~ 'cache1.example.org' ]] +nix verify -r "$fixedOutPath" +nix verify -r "$dependenciesOutPath" --sigs-needed 1 --trusted-public-keys $(cat $TEST_ROOT/pk1) diff --git a/tests/local.mk b/tests/local.mk index dd9a0ad56..01c35551f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -7,6 +7,7 @@ nix_tests = \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ + db-migration.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ From a0866c8ea4bc66f9aacc7ad19139d57946b3df18 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 13:43:08 +0100 Subject: [PATCH 193/725] Make the tests (optionnally) run in another derivation That way we can run them without rebuilding Nix --- flake.nix | 41 ++++++++++++++++++++++++++++++++++++----- tests/common.sh.in | 6 ++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 3e236aaca..c2e5db53a 100644 --- a/flake.nix +++ b/flake.nix @@ -144,6 +144,32 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; + testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { + NIX_DAEMON_PACKAGE = daemon; + NIX_CLIENT_PACKAGE = client; + name = "nix-tests-${client.version}-against-${daemon.version}"; + inherit version; + + src = self; + + VERSION_SUFFIX = versionSuffix; + + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ awsDeps; + propagatedBuildInputs = propagatedDeps; + + enableParallelBuilding = true; + + dontBuild = true; + doInstallCheck = true; + + installPhase = '' + mkdir -p $out + ''; + installCheckPhase = "make installcheck"; + + }; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -153,7 +179,7 @@ # An older version of Nix to test against when using the daemon. # Currently using `nixUnstable` as the stable one doesn't respect # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. - mainstream-nix = prev.nixUnstable; + nixStable = prev.nix; nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; @@ -163,8 +189,6 @@ VERSION_SUFFIX = versionSuffix; - OUTER_NIX = mainstream-nix; - outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; @@ -441,6 +465,15 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + installTests = + let pkgs = nixpkgsFor.${system}; in + pkgs.runCommand "install-tests" { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"; }); packages = forAllSystems (system: { @@ -493,8 +526,6 @@ stdenv.mkDerivation { name = "nix"; - OUTER_NIX = mainstream-nix; - outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps; diff --git a/tests/common.sh.in b/tests/common.sh.in index 277dd6dfa..d31d3fbb8 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -29,6 +29,12 @@ unset XDG_CACHE_HOME mkdir -p $TEST_HOME export PATH=@bindir@:$PATH +if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then + export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH +fi +if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then + export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon" +fi coreutils=@coreutils@ export dot=@dot@ From 81df1b5c687b7606f0159485c33bf5f7e2614eba Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 14:15:57 +0100 Subject: [PATCH 194/725] Remove the `remote-store-old-daemon` test Doesn't make sense anymore with the new setup --- tests/local.mk | 2 +- tests/remote-store-old-daemon.sh | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 tests/remote-store-old-daemon.sh diff --git a/tests/local.mk b/tests/local.mk index 01c35551f..e7e85f97e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -6,7 +6,7 @@ nix_tests = \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ - local-store.sh remote-store.sh remote-store-old-daemon.sh export.sh export-graph.sh \ + local-store.sh remote-store.sh export.sh export-graph.sh \ db-migration.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ diff --git a/tests/remote-store-old-daemon.sh b/tests/remote-store-old-daemon.sh deleted file mode 100644 index ede7ce716..000000000 --- a/tests/remote-store-old-daemon.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Test that the new Nix can properly talk to an old daemon. -# If `$OUTER_NIX` isn't set (e.g. when bootsraping), just skip this test - -if [[ -n "$OUTER_NIX" ]]; then - export NIX_DAEMON_COMMAND=$OUTER_NIX/bin/nix-daemon - source remote-store.sh -fi From be60c9ef50bf5fa653138802f63727fa0aadf50a Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 14:20:10 +0100 Subject: [PATCH 195/725] Fix the db-migration test --- tests/db-migration.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/db-migration.sh b/tests/db-migration.sh index e6a405770..e0ff7d311 100644 --- a/tests/db-migration.sh +++ b/tests/db-migration.sh @@ -1,7 +1,8 @@ # Test that we can successfully migrate from an older db schema # Only run this if we have an older Nix available -if [[ -z "$OUTER_NIX" ]]; then +# XXX: This assumes that the `daemon` package is older than the `client` one +if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then exit 0 fi @@ -9,7 +10,7 @@ source common.sh # Fill the db using the older Nix PATH_WITH_NEW_NIX="$PATH" -export PATH="$OUTER_NIX/bin:$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" clearStore nix-build simple.nix --no-out-link nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 From 5ec873b127139ca90cc31967c25c9a34fb4cc3e4 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Mar 2021 16:44:42 +0100 Subject: [PATCH 196/725] Shorten the test drv name To prevent the OSX build to fail because of a too long socket path --- flake.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index c2e5db53a..1cd54e702 100644 --- a/flake.nix +++ b/flake.nix @@ -147,7 +147,10 @@ testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - name = "nix-tests-${client.version}-against-${daemon.version}"; + # Must keep this name short as OSX has a rather strict limit on the + # socket path length, and this name appears in the path of the + # nix-daemon socket used in the tests + name = "nix-tests"; inherit version; src = self; From 77f5d171e17294ebb017a386d4408bf4613dfed7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Mar 2021 16:53:39 +0100 Subject: [PATCH 197/725] --override-input: Imply --no-write-lock-file Fixes #3779. --- src/libcmd/installables.cc | 3 ++- src/nix/flake.cc | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 7102f5a1a..898e642a5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -111,10 +111,11 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { + lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."))); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2f0c468a8..d37791aba 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -110,6 +110,7 @@ struct CmdFlakeUpdate : FlakeCommand removeFlag("recreate-lock-file"); removeFlag("update-input"); removeFlag("no-update-lock-file"); + removeFlag("no-write-lock-file"); } std::string doc() override @@ -124,6 +125,7 @@ struct CmdFlakeUpdate : FlakeCommand settings.tarballTtl = 0; lockFlags.recreateLockFile = true; + lockFlags.writeLockFile = true; lockFlake(); } @@ -136,6 +138,12 @@ struct CmdFlakeLock : FlakeCommand return "create missing lock file entries"; } + CmdFlakeLock() + { + /* Remove flags that don't make sense. */ + removeFlag("no-write-lock-file"); + } + std::string doc() override { return @@ -147,6 +155,8 @@ struct CmdFlakeLock : FlakeCommand { settings.tarballTtl = 0; + lockFlags.writeLockFile = true; + lockFlake(); } }; From 66fa1c7375e4b3073a16df4678cf1d37446ed20b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Mar 2021 17:19:04 +0100 Subject: [PATCH 198/725] Merge 'nix flake {info,list-inputs}' into 'nix flake metadata' Fixes #4613. --- src/nix/flake-list-inputs.md | 23 ---- src/nix/{flake-info.md => flake-metadata.md} | 27 ++-- src/nix/flake.cc | 134 ++++++++----------- tests/flakes.sh | 38 +++--- 4 files changed, 92 insertions(+), 130 deletions(-) delete mode 100644 src/nix/flake-list-inputs.md rename src/nix/{flake-info.md => flake-metadata.md} (75%) diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md deleted file mode 100644 index 250e13be0..000000000 --- a/src/nix/flake-list-inputs.md +++ /dev/null @@ -1,23 +0,0 @@ -R""( - -# Examples - -* Show the inputs of the `hydra` flake: - - ```console - # nix flake list-inputs github:NixOS/hydra - github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 - ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 - │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f - │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 - └───nixpkgs follows input 'nix/nixpkgs' - ``` - -# Description - -This command shows the inputs of the flake specified by the flake -referenced *flake-url*. Since it prints the locked inputs that result -from generating or updating the lock file, this command essentially -displays the contents of the flake's lock file in human-readable form. - -)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-metadata.md similarity index 75% rename from src/nix/flake-info.md rename to src/nix/flake-metadata.md index fda3171db..5a009409b 100644 --- a/src/nix/flake-info.md +++ b/src/nix/flake-metadata.md @@ -5,19 +5,24 @@ R""( * Show what `nixpkgs` resolves to: ```console - # nix flake info nixpkgs - Resolved URL: github:NixOS/nixpkgs - Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Description: A collection of packages for the Nix package manager - Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source - Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Last modified: 2020-12-23 12:36:12 + # nix flake metadata nixpkgs + Resolved URL: github:edolstra/dwarffs + Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 + Description: A filesystem that fetches DWARF debug info from the Internet on demand + Path: /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source + Revision: f691e2c991e75edb22836f1dbe632c40324215c5 + Last modified: 2021-01-21 15:41:26 + Inputs: + ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' ``` * Show information about `dwarffs` in JSON format: ```console - # nix flake info dwarffs --json | jq . + # nix flake metadata dwarffs --json | jq . { "description": "A filesystem that fetches DWARF debug info from the Internet on demand", "lastModified": 1597153508, @@ -29,6 +34,7 @@ R""( "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", "type": "github" }, + "locks": { ... }, "original": { "id": "dwarffs", "type": "indirect" @@ -75,6 +81,9 @@ data. This includes: time of the commit of the locked flake; for tarball flakes, it's the most recent timestamp of any file inside the tarball. +* `Inputs`: The flake inputs with their corresponding lock file + entries. + With `--json`, the output is a JSON object with the following fields: * `original` and `originalUrl`: The flake reference specified by the @@ -96,4 +105,6 @@ With `--json`, the output is a JSON object with the following fields: * `lastModified`: See `Last modified` above. +* `locks`: The contents of `flake.lock`. + )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d37791aba..5ce2e082c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -43,12 +43,6 @@ public: return parseFlakeRef(flakeUrl, absPath(".")); //FIXME } - Flake getFlake() - { - auto evalState = getEvalState(); - return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); - } - LockedFlake lockFlake() { return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); @@ -60,43 +54,6 @@ public: } }; -static void printFlakeInfo(const Store & store, const Flake & flake) -{ - logger->cout("Resolved URL: %s", flake.resolvedRef.to_string()); - logger->cout("Locked URL: %s", flake.lockedRef.to_string()); - if (flake.description) - logger->cout("Description: %s", *flake.description); - logger->cout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input.getRev()) - logger->cout("Revision: %s", rev->to_string(Base16, false)); - if (auto revCount = flake.lockedRef.input.getRevCount()) - logger->cout("Revisions: %s", *revCount); - if (auto lastModified = flake.lockedRef.input.getLastModified()) - logger->cout("Last modified: %s", - std::put_time(std::localtime(&*lastModified), "%F %T")); -} - -static nlohmann::json flakeToJSON(const Store & store, const Flake & flake) -{ - nlohmann::json j; - if (flake.description) - j["description"] = *flake.description; - j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); - j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); - j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); - if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); - if (auto revCount = flake.lockedRef.input.getRevCount()) - j["revCount"] = *revCount; - if (auto lastModified = flake.lockedRef.input.getLastModified()) - j["lastModified"] = *lastModified; - j["path"] = store.printStorePath(flake.sourceInfo->storePath); - return j; -} - struct CmdFlakeUpdate : FlakeCommand { std::string description() override @@ -175,54 +132,72 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, callback(attr.name, *attr.value, *attr.pos); } -struct CmdFlakeInfo : FlakeCommand, MixJSON +struct CmdFlakeMetadata : FlakeCommand, MixJSON { std::string description() override { - return "list info about a given flake"; + return "show flake metadata"; } std::string doc() override { return - #include "flake-info.md" + #include "flake-metadata.md" ; } void run(nix::ref store) override { - auto flake = getFlake(); + auto lockedFlake = lockFlake(); + auto & flake = lockedFlake.flake; if (json) { - auto json = flakeToJSON(*store, flake); - logger->cout("%s", json.dump()); - } else - printFlakeInfo(*store, flake); - } -}; + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["locks"] = lockedFlake.lockFile.toJSON(); + logger->cout("%s", j.dump()); + } else { + logger->cout( + ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", + flake.resolvedRef.to_string()); + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); + if (flake.description) + logger->cout( + ANSI_BOLD "Description:" ANSI_NORMAL " %s", + *flake.description); + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->cout( + ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", + *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->cout( + ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); -struct CmdFlakeListInputs : FlakeCommand, MixJSON -{ - std::string description() override - { - return "list flake inputs"; - } - - std::string doc() override - { - return - #include "flake-list-inputs.md" - ; - } - - void run(nix::ref store) override - { - auto flake = lockFlake(); - - if (json) - logger->cout("%s", flake.lockFile.toJSON()); - else { - logger->cout("%s", flake.flake.lockedRef); + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; @@ -236,7 +211,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); + *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); bool firstVisit = visited.insert(*lockedNode).second; @@ -249,8 +224,8 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON } }; - visited.insert(flake.lockFile.root); - recurse(*flake.lockFile.root, ""); + visited.insert(lockedFlake.lockFile.root); + recurse(*lockedFlake.lockFile.root, ""); } } }; @@ -1048,8 +1023,7 @@ struct CmdFlake : NixMultiCommand : MultiCommand({ {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, - {"info", []() { return make_ref(); }}, - {"list-inputs", []() { return make_ref(); }}, + {"metadata", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, {"init", []() { return make_ref(); }}, {"new", []() { return make_ref(); }}, diff --git a/tests/flakes.sh b/tests/flakes.sh index 9747aba7a..e78e4a39d 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -164,16 +164,17 @@ EOF # Test 'nix flake list'. [[ $(nix registry list | wc -l) == 7 ]] -# Test 'nix flake info'. -nix flake info flake1 | grep -q 'URL: .*flake1.*' +# Test 'nix flake metadata'. +nix flake metadata flake1 +nix flake metadata flake1 | grep -q 'Locked URL:.*flake1.*' -# Test 'nix flake info' on a local flake. -(cd $flake1Dir && nix flake info) | grep -q 'URL: .*flake1.*' -(cd $flake1Dir && nix flake info .) | grep -q 'URL: .*flake1.*' -nix flake info $flake1Dir | grep -q 'URL: .*flake1.*' +# Test 'nix flake metadata' on a local flake. +(cd $flake1Dir && nix flake metadata) | grep -q 'URL:.*flake1.*' +(cd $flake1Dir && nix flake metadata .) | grep -q 'URL:.*flake1.*' +nix flake metadata $flake1Dir | grep -q 'URL:.*flake1.*' -# Test 'nix flake info --json'. -json=$(nix flake info flake1 --json | jq .) +# Test 'nix flake metadata --json'. +json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] @@ -181,7 +182,7 @@ hash1=$(echo "$json" | jq -r .revision) echo -n '# foo' >> $flake1Dir/flake.nix git -C $flake1Dir commit -a -m 'Foo' -hash2=$(nix flake info flake1 --json --refresh | jq -r .revision) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. @@ -630,7 +631,7 @@ hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit' nix build -o $TEST_ROOT/result hg+file://$flake5Dir [[ -e $TEST_ROOT/result/hello ]] -(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision) nix eval hg+file://$flake5Dir#expr @@ -638,13 +639,13 @@ nix eval hg+file://$flake5Dir#expr (! nix eval hg+file://$flake5Dir#expr --no-allow-dirty) -(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revision) hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file' -nix flake info --json hg+file://$flake5Dir --refresh | jq -e -r .revision -nix flake info --json hg+file://$flake5Dir -[[ $(nix flake info --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]] +nix flake metadata --json hg+file://$flake5Dir --refresh | jq -e -r .revision +nix flake metadata --json hg+file://$flake5Dir +[[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]] nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty @@ -654,7 +655,7 @@ tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5 nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz # Building with a tarball URL containing a SRI hash should also work. -url=$(nix flake info --json file://$TEST_ROOT/flake.tar.gz | jq -r .url) +url=$(nix flake metadata --json file://$TEST_ROOT/flake.tar.gz | jq -r .url) [[ $url =~ sha256- ]] nix build -o $TEST_ROOT/result $url @@ -680,9 +681,8 @@ nix flake lock $flake3Dir nix flake lock $flake3Dir --update-input flake2/flake1 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] -# Test 'nix flake list-inputs'. -[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]] -nix flake list-inputs $flake3Dir --json | jq . +# Test 'nix flake metadata --json'. +nix flake metadata $flake3Dir --json | jq . # Test circular flake dependencies. cat > $flakeA/flake.nix < Date: Tue, 16 Mar 2021 18:51:17 +0100 Subject: [PATCH 199/725] Fix Nix to properly work with stores using a scoped IPv6 address According to RFC4007[1], IPv6 addresses can have a so-called zone_id separated from the actual address with `%` as delimiter. In contrast to Nix 2.3, the version on `master` doesn't recognize it as such: $ nix ping-store --store ssh://root@fe80::1%18 --experimental-features nix-command warning: 'ping-store' is a deprecated alias for 'store ping' error: --- Error ----------------------------------------------------------------- nix don't know how to open Nix store 'ssh://root@fe80::1%18' I modified the IPv6 match-regex accordingly to optionally detect this part of the address. As we don't seem to do anything special with it, I decided to leave it as part of the URL for now. Fixes #4490 [1] https://tools.ietf.org/html/rfc4007 --- src/libutil/tests/url.cc | 18 ++++++++++++++++++ src/libutil/url-parts.hh | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index 80646ad3e..aff58e9ee 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -117,6 +117,24 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parseScopedRFC4007IPv6Address) { + auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .scheme = "http", + .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + TEST(parseURL, parseIPv6Address) { auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; auto parsed = parseURL(s); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 862d9fa6e..da10a6bbc 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; -const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+"; +const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; From a61112aadf58c1578cbdcba32b1582d25ca7ed9b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Mar 2021 11:27:11 +0100 Subject: [PATCH 200/725] Remove unimplemented hashAlgoOpt It was in the header but never implemented. --- src/libstore/derivations.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 061d70f69..2df440536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -52,7 +52,7 @@ struct DerivationOutput DerivationOutputCAFloating, DerivationOutputDeferred > output; - std::optional hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer interface provided by BasicDerivation::outputsAndOptPaths */ From ef83ced4e170130cb6f9acd1d253351b02490658 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Mar 2021 17:21:37 +0100 Subject: [PATCH 201/725] Restore 'nix flake info' as a deprecated alias --- src/nix/flake.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5ce2e082c..a2b6c0303 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -230,6 +230,15 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON } }; +struct CmdFlakeInfo : CmdFlakeMetadata +{ + void run(nix::ref store) override + { + warn("'nix flake info' is a deprecated alias for 'nix flake metadata'"); + CmdFlakeMetadata::run(store); + } +}; + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -1024,6 +1033,7 @@ struct CmdFlake : NixMultiCommand {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, {"init", []() { return make_ref(); }}, {"new", []() { return make_ref(); }}, From 3e0e443181997c52b0db19ae781948c573a634dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Mar 2021 17:56:39 +0100 Subject: [PATCH 202/725] ProgressBar: Respect verbosity level This makes its behaviour consistent with SimpleLogger. --- src/libmain/progress-bar.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 0e5432fca..15354549a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -122,6 +122,7 @@ public: void log(Verbosity lvl, const FormatOrString & fs) override { + if (lvl > verbosity) return; auto state(state_.lock()); log(*state, lvl, fs.s); } From 1765711b68c8647b502c2c009dace9632e9300d7 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 18:43:37 -0400 Subject: [PATCH 203/725] tests/config: Fix config test configuration First, "XDG_CONFIG_HOME" shouldn't be named "home", as it may be confusing compared with `$HOME`, which an upcoming test will be using. Then, using a fixed location for the test is problematic. Use `$TEST_ROOT` instead. --- tests/config.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/config.sh b/tests/config.sh index eaa46c395..1edc09c1a 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -1,15 +1,15 @@ source common.sh # Test that files are loaded from XDG by default -export XDG_CONFIG_HOME=/tmp/home -export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +export XDG_CONFIG_HOME=$TEST_ROOT/confighome +export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) -[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] +[[ $files == "$TEST_ROOT/confighome/nix/nix.conf:$TEST_ROOT/dir1/nix/nix.conf:$TEST_ROOT/dir2/nix/nix.conf" ]] # Test that setting NIX_USER_CONF_FILES overrides all the default user config files -export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +export NIX_USER_CONF_FILES=$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) -[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] +[[ $files == "$TEST_ROOT/file1.conf:$TEST_ROOT/file2.conf" ]] # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") @@ -24,4 +24,4 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "nix-command flakes" ]] \ No newline at end of file +[[ $exp_features == "nix-command flakes" ]] From bf07581497d55ade85d80e5d9ad9bf5d962e3403 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 19:02:11 -0400 Subject: [PATCH 204/725] tests: Test `.config` stays clean with XDG_CONFIG_HOME set --- tests/config.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/config.sh b/tests/config.sh index 1edc09c1a..01c78f2c3 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -1,5 +1,31 @@ source common.sh +# Isolate the home for this test. +# Other tests (e.g. flake registry tests) could be writing to $HOME in parallel. +export HOME=$TEST_ROOT/userhome + +# Test that using XDG_CONFIG_HOME works +# Assert the config folder didn't exist initially. +[ ! -e "$HOME/.config" ] +# Without XDG_CONFIG_HOME, creates $HOME/.config +unset XDG_CONFIG_HOME +# Run against the nix registry to create the config dir +# (Tip: this relies on removing non-existent entries being a no-op!) +nix registry remove userhome-without-xdg +# Verifies it created it +[ -e "$HOME/.config" ] +# Remove the directory it created +rm -rf "$HOME/.config" +# Run the same test, but with XDG_CONFIG_HOME +export XDG_CONFIG_HOME=$TEST_ROOT/confighome +# Assert the XDG_CONFIG_HOME/nix path does not exist yet. +[ ! -e "$TEST_ROOT/confighome/nix" ] +nix registry remove userhome-with-xdg +# Verifies the confighome path has been created +[ -e "$TEST_ROOT/confighome/nix" ] +# Assert the .config folder hasn't been created. +[ ! -e "$HOME/.config" ] + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 From 66b857244ff062f6bb97c23e2423338ad242f7a1 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Wed, 17 Mar 2021 17:56:57 -0400 Subject: [PATCH 205/725] Use the appropriate config dir for the registry --- src/libfetchers/registry.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 81b2227de..74376adc0 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -114,7 +114,7 @@ static std::shared_ptr getSystemRegistry() Path getUserRegistryPath() { - return getHome() + "/.config/nix/registry.json"; + return getConfigDir() + "/nix/registry.json"; } std::shared_ptr getUserRegistry() From 9d309de0de9a09d36717abd02a66b51815397d66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Feb 2021 18:42:46 +0000 Subject: [PATCH 206/725] Clean up serialization for `BuildResult` A few versioning mistakes were corrected: - In 27b5747ca7b5599768083dde5fa4d36bfbb0f66f, Daemon protocol had some version `>= 0xc` that should have been `>= 0x1c`, or `28` since the other conditions used decimal. - In a2b69660a9b326b95d48bd222993c5225bbd5b5f, legacy SSH gated new CAS info on version 6, but version 5 in the server. It is now 6 everywhere. Additionally, legacy ssh was sending over more metadata than the daemon one was. The daemon now sends that data too. CC @regnat Co-authored-by: Cole Helbling --- src/libstore/daemon.cc | 5 ++++- src/libstore/remote-store.cc | 20 ++++++++++++++------ src/libstore/serve-protocol.hh | 2 +- src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 2 +- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ba7959263..dc9cd2cbd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -575,7 +575,10 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; + } + if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { worker_proto::write(*store, to, res.builtOutputs); } break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0d884389a..b01cb5a62 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -62,9 +62,15 @@ void write(const Store & store, Sink & out, const Realisation & realisation) { out << realisation.toJSON().dump(); } DrvOutput read(const Store & store, Source & from, Phantom _) -{ return DrvOutput::parse(readString(from)); } +{ + return DrvOutput::parse(readString(from)); +} + void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ out << drvOutput.to_string(); } +{ + out << drvOutput.to_string(); +} + std::optional read(const Store & store, Source & from, Phantom> _) { @@ -677,10 +683,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->to << buildMode; conn.processStderr(); BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) { + res.status = (BuildResult::Status) readInt(conn->from); + conn->from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); res.builtOutputs = builtOutputs; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 0a17387cb..02d0810cc 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x206 +#define SERVE_PROTOCOL_VERSION (2 << 8 | 6) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 95f08bc9a..be071dd78 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11c +#define PROTOCOL_VERSION (1 << 8 | 29) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94d4881dd..b684feccb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -905,7 +905,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 5)) { + if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { worker_proto::write(*store, out, status.builtOutputs); } From f44206e71953501af502354ab1c747aa2412d676 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Mar 2021 15:18:48 +0000 Subject: [PATCH 207/725] Fix typos in the last PR #4656 --- src/libstore/daemon.cc | 2 +- src/libstore/remote-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index dc9cd2cbd..f28ab6438 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -576,7 +576,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->stopWork(); to << res.status << res.errorMsg; if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - out << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; + to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { worker_proto::write(*store, to, res.builtOutputs); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b01cb5a62..ccf095dc2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,7 +686,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); From 0f40561c78bb5566b42d054620b0576e14fe4627 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 23 Mar 2021 10:19:00 +0700 Subject: [PATCH 208/725] nix.conf builders: refer to manual page --- src/libstore/globals.hh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index bf0767dfa..3e4ead76c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -207,15 +207,8 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", R"( - A semicolon-separated list of build machines, where each machine follows this format: - - {protocol}://{user}@{host} [{comma sep. systems} - {maxJobs} {speedFactor} {comma sep. features}] - - Examples: - - ssh://root@builder1.com - - ssh://root@builder2.com x86_64-linux,aarch64-linux - 40 20 nixos-test,benchmark,big-parallel,kvm + A semicolon-separated list of build machines. + For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; Setting buildersUseSubstitutes{ From 71f92741ec979c1059938a638b7fc8da6d7b0936 Mon Sep 17 00:00:00 2001 From: Nicolas Stig124 FORMICHELLA Date: Tue, 23 Mar 2021 16:23:24 +0100 Subject: [PATCH 209/725] Added Debian-based OS's profiles --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5e8b4ac18..4cc11d210 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -31,7 +31,7 @@ readonly NIX_FIRST_BUILD_UID="30001" readonly NIX_ROOT="/nix" readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" From 4638bcfb2cfb74cb5029c0da0af38bb7ca4b4a6f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Mar 2021 16:14:38 +0100 Subject: [PATCH 210/725] Fix some typos Fixes #4671. --- doc/manual/src/command-ref/nix-store.md | 2 +- src/libexpr/flake/flake.hh | 2 +- src/libstore/globals.cc | 2 +- src/libutil/config.cc | 28 +++++++++++----------- src/libutil/config.hh | 22 ++++++++--------- src/libutil/tests/config.cc | 32 ++++++++++++------------- src/libutil/util.cc | 2 +- src/nix/build.md | 2 +- src/nix/flake-init.md | 2 +- src/nix/flake.md | 4 ++-- src/nix/main.cc | 8 +++---- src/nix/store-prefetch-file.md | 2 +- 12 files changed, 54 insertions(+), 54 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 361c20cc9..49d06f31e 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -79,7 +79,7 @@ paths. Realisation is a somewhat overloaded term: system). If the path is already valid, we are done immediately. Otherwise, the path and any missing paths in its closure may be produced through substitutes. If there are no (successful) - subsitutes, realisation fails. + substitutes, realisation fails. The output path of each derivation is printed on standard output. (For non-derivations argument, the argument itself is printed.) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 65ed1ad0a..d17d5e183 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -113,7 +113,7 @@ struct LockFlags /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; - /* Flake inputs to be overriden. */ + /* Flake inputs to be overridden. */ std::map inputOverrides; /* Flake inputs to be updated. This means that any existing lock diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8d44003f4..d3b27d7be 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -81,7 +81,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverriden(); + globalConfig.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7467e5ac0..bda07cd55 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -20,7 +20,7 @@ bool Config::set(const std::string & name, const std::string & value) return false; } i->second.setting->set(value, append); - i->second.setting->overriden = true; + i->second.setting->overridden = true; return true; } @@ -35,7 +35,7 @@ void Config::addSetting(AbstractSetting * setting) auto i = unknownSettings.find(setting->name); if (i != unknownSettings.end()) { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -48,7 +48,7 @@ void Config::addSetting(AbstractSetting * setting) alias, setting->name); else { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -69,10 +69,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -void Config::getSettings(std::map & res, bool overridenOnly) +void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -136,10 +136,10 @@ void AbstractConfig::applyConfigFile(const Path & path) } catch (SysError &) { } } -void Config::resetOverriden() +void Config::resetOverridden() { for (auto & s : _settings) - s.second.setting->overriden = false; + s.second.setting->overridden = false; } nlohmann::json Config::toJSON() @@ -169,7 +169,7 @@ AbstractSetting::AbstractSetting( void AbstractSetting::setDefault(const std::string & str) { - if (!overriden) set(str); + if (!overridden) set(str); } nlohmann::json AbstractSetting::toJSON() @@ -203,7 +203,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s); }}, + .handler = {[=](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -212,7 +212,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s, true); }}, + .handler = {[=](std::string s) { overridden = true; set(s, true); }}, }); } @@ -365,16 +365,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value) return false; } -void GlobalConfig::getSettings(std::map & res, bool overridenOnly) +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) { for (auto & config : *configRegistrations) - config->getSettings(res, overridenOnly); + config->getSettings(res, overriddenOnly); } -void GlobalConfig::resetOverriden() +void GlobalConfig::resetOverridden() { for (auto & config : *configRegistrations) - config->resetOverriden(); + config->resetOverridden(); } nlohmann::json GlobalConfig::toJSON() diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 71e31656d..bf81b4892 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -71,9 +71,9 @@ public: /** * Adds the currently known settings to the given result map `res`. * - res: map to store settings in - * - overridenOnly: when set to true only overridden settings will be added to `res` + * - overriddenOnly: when set to true only overridden settings will be added to `res` */ - virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + virtual void getSettings(std::map & res, bool overriddenOnly = false) = 0; /** * Parses the configuration in `contents` and applies it @@ -91,7 +91,7 @@ public: /** * Resets the `overridden` flag of all Settings */ - virtual void resetOverriden() = 0; + virtual void resetOverridden() = 0; /** * Outputs all settings to JSON @@ -127,7 +127,7 @@ public: MyClass() : Config(readConfigFile("/etc/my-app.conf")) { - std::cout << foo << "\n"; // will print 123 unless overriden + std::cout << foo << "\n"; // will print 123 unless overridden } }; */ @@ -163,9 +163,9 @@ public: void addSetting(AbstractSetting * setting); - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; @@ -184,7 +184,7 @@ public: int created = 123; - bool overriden = false; + bool overridden = false; void setDefault(const std::string & str); @@ -215,7 +215,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverriden() const { return overriden; } + bool isOverridden() const { return overridden; } }; /* A setting of type T. */ @@ -252,7 +252,7 @@ public: virtual void override(const T & v) { - overriden = true; + overridden = true; value = v; } @@ -324,9 +324,9 @@ struct GlobalConfig : public AbstractConfig bool set(const std::string & name, const std::string & value) override; - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c305af9f5..0ebdaf3db 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -29,20 +29,20 @@ namespace nix { std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, ""); ASSERT_EQ(iter->second.description, "description\n"); } - TEST(Config, getDefinedOverridenSettingNotSet) { + TEST(Config, getDefinedOverriddenSettingNotSet) { Config config; std::string value; std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); const auto e = settings.find("name-of-the-setting"); ASSERT_EQ(e, settings.end()); } @@ -55,7 +55,7 @@ namespace nix { setting.assign("value"); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, "value"); @@ -69,7 +69,7 @@ namespace nix { ASSERT_TRUE(config.set("name-of-the-setting", "value")); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto e = settings.find("name-of-the-setting"); ASSERT_NE(e, settings.end()); ASSERT_EQ(e->second.value, "value"); @@ -100,7 +100,7 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings.find("key"), settings.end()); } @@ -108,17 +108,17 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings["key"].value, "value"); } } - TEST(Config, resetOverriden) { + TEST(Config, resetOverridden) { Config config; - config.resetOverriden(); + config.resetOverridden(); } - TEST(Config, resetOverridenWithSetting) { + TEST(Config, resetOverriddenWithSetting) { Config config; Setting setting{&config, "", "name-of-the-setting", "description"}; @@ -127,7 +127,7 @@ namespace nix { setting.set("foo"); ASSERT_EQ(setting.get(), "foo"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } @@ -135,18 +135,18 @@ namespace nix { std::map settings; setting.override("bar"); - ASSERT_TRUE(setting.overriden); + ASSERT_TRUE(setting.overridden); ASSERT_EQ(setting.get(), "bar"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_FALSE(settings.empty()); } { std::map settings; - config.resetOverriden(); - ASSERT_FALSE(setting.overriden); - config.getSettings(settings, /* overridenOnly = */ true); + config.resetOverridden(); + ASSERT_FALSE(setting.overridden); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ef37275ac..dea9c74b7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1590,7 +1590,7 @@ void startSignalHandlerThread() updateWindowSize(); if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("quering signal mask"); + throw SysError("querying signal mask"); sigset_t set; sigemptyset(&set); diff --git a/src/nix/build.md b/src/nix/build.md index c2f3e387a..20138b7e0 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -81,7 +81,7 @@ path installables are substituted. Unless `--no-link` is specified, after a successful build, it creates symlinks to the store paths of the installables. These symlinks have -the prefix `./result` by default; this can be overriden using the +the prefix `./result` by default; this can be overridden using the `--out-link` option. Each symlink has a suffix `--`, where *N* is the index of the installable (with the left-most installable having index 0), and *outname* is the symbolic derivation output name diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index c66154ad5..890038016 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -24,7 +24,7 @@ R""( This command creates a flake in the current directory by copying the files of a template. It will not overwrite existing files. The default -template is `templates#defaultTemplate`, but this can be overriden +template is `templates#defaultTemplate`, but this can be overridden using `-t`. # Template definitions diff --git a/src/nix/flake.md b/src/nix/flake.md index 440c45dd1..0035195e5 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -70,7 +70,7 @@ Here are some examples of flake references in their URL-like representation: * `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` - entry in the flake registry, with its Git revision overriden to a + entry in the flake registry, with its Git revision overridden to a specific value. * `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` repository on GitHub. @@ -377,7 +377,7 @@ outputs = { self, nixpkgs, grcov }: { }; ``` -Transitive inputs can be overriden from a `flake.nix` file. For +Transitive inputs can be overridden from a `flake.nix` file. For example, the following overrides the `nixpkgs` input of the `nixops` input: diff --git a/src/nix/main.cc b/src/nix/main.cc index 06e221682..f8701ee56 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -309,13 +309,13 @@ void mainWrapped(int argc, char * * argv) if (!args.useNet) { // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overriden) + if (!settings.useSubstitutes.overridden) settings.useSubstitutes = false; - if (!settings.tarballTtl.overriden) + if (!settings.tarballTtl.overridden) settings.tarballTtl = std::numeric_limits::max(); - if (!fileTransferSettings.tries.overriden) + if (!fileTransferSettings.tries.overridden) fileTransferSettings.tries = 0; - if (!fileTransferSettings.connectTimeout.overriden) + if (!fileTransferSettings.connectTimeout.overridden) fileTransferSettings.connectTimeout = 1; } diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md index 1663b847b..f9fdcbc57 100644 --- a/src/nix/store-prefetch-file.md +++ b/src/nix/store-prefetch-file.md @@ -27,6 +27,6 @@ the resulting store path and the cryptographic hash of the contents of the file. The name component of the store path defaults to the last component of -*url*, but this can be overriden using `--name`. +*url*, but this can be overridden using `--name`. )"" From dd77f71afe6733e9790dd001125c423cb648b7ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Mar 2021 17:10:15 +0100 Subject: [PATCH 211/725] LocalBinaryCacheStore::upsertFile(): Fix race When multiple threads try to upsert the same file, this could fail. Fixes #4667. --- src/libstore/local-binary-cache-store.cc | 5 ++++- tests/ca/substitute.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a58b7733f..964c4017e 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,8 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" +#include + namespace nix { struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig @@ -50,7 +52,8 @@ protected: const std::string & mimeType) override { auto path2 = binaryCacheDir + "/" + path; - Path tmp = path2 + ".tmp." + std::to_string(getpid()); + static std::atomic counter{0}; + Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index 79a6ef8b1..b44fe499a 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -6,6 +6,8 @@ source common.sh sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf +rm -rf $TEST_ROOT/binary_cache + export REMOTE_STORE=file://$TEST_ROOT/binary_cache buildDrvs () { @@ -13,6 +15,7 @@ buildDrvs () { } # Populate the remote cache +clearStore buildDrvs --post-build-hook ../push-to-store.sh # Restart the build on an empty store, ensuring that we don't build From ce791535f63502215d3d41b6ca8d9e62c5fb72e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Mar 2021 14:54:05 +0200 Subject: [PATCH 212/725] nixpkgs/master compatibility --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e59ec9a35..925017472 100644 --- a/flake.nix +++ b/flake.nix @@ -90,7 +90,7 @@ lowdown gmock ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isx86_64 libcpuid; From edd606ae62e213c2a30ff76b8eea4f75ea703d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Sat, 27 Mar 2021 14:15:28 +0100 Subject: [PATCH 213/725] fetchGit: don't prefix "refs/heads/" on ref = "HEAD" This fixes builtins.fetchGit { url = ...; ref = "HEAD"; }, that works in stable nix (v2.3.10), but is broken in nix master: $ ./result/bin/nix repl Welcome to Nix version 2.4pre19700101_dd77f71. Type :? for help. nix-repl> builtins.fetchGit { url = "https://github.com/NixOS/nix"; ref = "HEAD"; } fetching Git repository 'https://github.com/NixOS/nix'fatal: couldn't find remote ref refs/heads/HEAD error: program 'git' failed with exit code 128 The documentation for builtins.fetchGit says ref = "HEAD" is the default, so it should also be supported to explicitly pass it. I came across this issue because poetry2nix can use ref = "HEAD" in some situations. Fixes #4674. --- src/libfetchers/git.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4f9db1bcd..b9a240b13 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -365,7 +365,9 @@ struct GitInputScheme : InputScheme ? "refs/*" : ref->compare(0, 5, "refs/") == 0 ? *ref - : "refs/heads/" + *ref; + : ref == "HEAD" + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; From f2a799b16d193a651f682da3ad2103c20ac82d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 30 Mar 2021 11:39:37 +0200 Subject: [PATCH 214/725] tests: check that builtins.fetchGit { ..., ref = "HEAD"; } works --- tests/fetchGit.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 1e8963d76..88744ee7f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -179,3 +179,13 @@ git clone --depth 1 file://$repo $TEST_ROOT/shallow path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") [[ $path3 = $path6 ]] [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] + +# Explicit ref = "HEAD" should work, and produce the same outPath as without ref +path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") +path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") +[[ $path7 = $path8 ]] + +# ref = "HEAD" should fetch the HEAD revision +rev4=$(git -C $repo rev-parse HEAD) +rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev") +[[ $rev4 = $rev4_nix ]] From f3f228700a52857fe6e8632df4e935551ea219ff Mon Sep 17 00:00:00 2001 From: Mykola Orliuk Date: Wed, 31 Mar 2021 04:20:41 +0200 Subject: [PATCH 215/725] canonPath in one pass --- src/libutil/util.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index dea9c74b7..c092076f3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -143,16 +143,18 @@ Path canonPath(const Path & path, bool resolveSymlinks) s += '/'; while (i != end && *i != '/') s += *i++; - /* If s points to a symlink, resolve it and restart (since - the symlink target might contain new symlinks). */ + /* If s points to a symlink, resolve it and continue from there */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", path); - temp = absPath(readLink(s), dirOf(s)) - + string(i, end); - i = temp.begin(); /* restart */ + temp = readLink(s) + string(i, end); + i = temp.begin(); end = temp.end(); - s = ""; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + } } } } From f66fb5fb5b1478a5da39d0e9cc0f835272199c5d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Mar 2021 12:06:43 +0100 Subject: [PATCH 216/725] flake.nix: Build nix with strictDeps = true --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 58dc5019d..adb796a05 100644 --- a/flake.nix +++ b/flake.nix @@ -233,6 +233,8 @@ separateDebugInfo = true; + strictDeps = true; + passthru.perl-bindings = with final; stdenv.mkDerivation { name = "nix-perl-${version}"; @@ -517,6 +519,8 @@ installCheckFlags = "sysconfdir=$(out)/etc"; stripAllList = ["bin"]; + + strictDeps = true; }; }); From c3090bc6fdf6e052cd4c56fce6aeb11ddeb5dd6f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Mar 2021 14:44:20 +0100 Subject: [PATCH 217/725] tests/*: show when tests are skipped --- tests/build-remote.sh | 4 ++-- tests/gc-runtime.sh | 2 +- tests/linux-sandbox.sh | 4 ++-- tests/recursive.sh | 2 +- tests/shell.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 04848e4b5..70f82e939 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,5 +1,5 @@ -if ! canUseSandbox; then exit; fi -if ! [[ $busybox =~ busybox ]]; then exit; fi +if ! canUseSandbox; then exit 99; fi +if ! [[ $busybox =~ busybox ]]; then exit 99; fi unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/gc-runtime.sh b/tests/gc-runtime.sh index 4c5028005..6094959cb 100644 --- a/tests/gc-runtime.sh +++ b/tests/gc-runtime.sh @@ -4,7 +4,7 @@ case $system in *linux*) ;; *) - exit 0; + exit 99; esac set -m # enable job control, needed for kill diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 70a90a907..eac62d461 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -2,13 +2,13 @@ source common.sh clearStore -if ! canUseSandbox; then exit; fi +if ! canUseSandbox; then exit 99; fi # Note: we need to bind-mount $SHELL into the chroot. Currently we # only support the case where $SHELL is in the Nix store, because # otherwise things get complicated (e.g. if it's in /bin, do we need # /lib as well?). -if [[ ! $SHELL =~ /nix/store ]]; then exit; fi +if [[ ! $SHELL =~ /nix/store ]]; then exit 99; fi chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 diff --git a/tests/recursive.sh b/tests/recursive.sh index b020ec710..a55b061b5 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -1,7 +1,7 @@ source common.sh # FIXME -if [[ $(uname) != Linux ]]; then exit; fi +if [[ $(uname) != Linux ]]; then exit 99; fi clearStore diff --git a/tests/shell.sh b/tests/shell.sh index 7a9ee8ab0..2b85bb337 100644 --- a/tests/shell.sh +++ b/tests/shell.sh @@ -6,7 +6,7 @@ clearCache nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' -if ! canUseSandbox; then exit; fi +if ! canUseSandbox; then exit 99; fi chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 From ff1a2143aa1338ccba0e2bc5ccd66bd3df8baa31 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Mar 2021 14:50:15 +0100 Subject: [PATCH 218/725] flake.nix: Make the sandbox tests work again --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index adb796a05..9a758eafa 100644 --- a/flake.nix +++ b/flake.nix @@ -78,7 +78,8 @@ buildPackages.git buildPackages.mercurial buildPackages.jq - ]; + ] + ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)]; buildDeps = [ curl @@ -90,7 +91,7 @@ lowdown gmock ] - ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)] + ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isx86_64 libcpuid; From 5926200db09ca4d0c5769edf24a3cf2e9f472d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?= <51823984+cgohla@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:54:09 +0100 Subject: [PATCH 219/725] [prerequisites]: add JSON lib dependency --- doc/manual/src/installation/prerequisites-source.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 6825af707..12758c5e1 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -69,3 +69,6 @@ `--disable-seccomp-sandboxing` option to the `configure` script (Not recommended unless your system doesn't support `libseccomp`). To get the library, visit . + + - Niels Lohmann's [JSON library](https://github.com/nlohmann/json). + From 00f00a995458776e33fdda692abe2099196ac566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 Apr 2021 21:32:09 +0200 Subject: [PATCH 220/725] bump actions --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2531a7d35..33035ca1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,9 +13,9 @@ jobs: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v12 + - uses: cachix/install-nix-action@v13 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v8 + - uses: cachix/cachix-action@v9 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -44,8 +44,8 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v12 - - uses: cachix/cachix-action@v8 + - uses: cachix/install-nix-action@v13 + - uses: cachix/cachix-action@v9 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -62,7 +62,7 @@ jobs: steps: - uses: actions/checkout@v2.3.4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@master + - uses: cachix/install-nix-action@v13 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" From f7d9f7c3381acef38e4db2bb2f9e0287c289be54 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Mar 2021 05:48:01 +0000 Subject: [PATCH 221/725] Pull out Buildable into its own file/header in libnixstore --- src/libcmd/installables.cc | 25 ------------------------- src/libcmd/installables.hh | 22 +--------------------- src/libstore/buildable.cc | 33 +++++++++++++++++++++++++++++++++ src/libstore/buildable.hh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 46 deletions(-) create mode 100644 src/libstore/buildable.cc create mode 100644 src/libstore/buildable.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 898e642a5..ca416b9ee 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -20,31 +20,6 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { - nlohmann::json res; - res["path"] = store->printStorePath(path); - return res; -} - -nlohmann::json BuildableFromDrv::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; - } - return res; -} - -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { - auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable); - } - return res; -} - void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b714f097b..d31afd3d5 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,13 +2,12 @@ #include "util.hh" #include "path.hh" +#include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" #include -#include - namespace nix { struct DrvInfo; @@ -16,25 +15,6 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct BuildableOpaque { - StorePath path; - nlohmann::json toJSON(ref store) const; -}; - -struct BuildableFromDrv { - StorePath drvPath; - std::map> outputs; - nlohmann::json toJSON(ref store) const; -}; - -typedef std::variant< - BuildableOpaque, - BuildableFromDrv -> Buildable; - -typedef std::vector Buildables; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); - struct App { std::vector context; diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc new file mode 100644 index 000000000..5cba45b1d --- /dev/null +++ b/src/libstore/buildable.cc @@ -0,0 +1,33 @@ +#include "buildable.hh" +#include "store-api.hh" + +#include + +namespace nix { + +nlohmann::json BuildableOpaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuildableFromDrv::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = path ? store->printStorePath(*path) : ""; + } + return res; +} + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const Buildable & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable); + } + return res; +} + +} diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh new file mode 100644 index 000000000..6177237be --- /dev/null +++ b/src/libstore/buildable.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "util.hh" +#include "path.hh" + +#include + +#include + +namespace nix { + +class Store; + +struct BuildableOpaque { + StorePath path; + nlohmann::json toJSON(ref store) const; +}; + +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + nlohmann::json toJSON(ref store) const; +}; + +typedef std::variant< + BuildableOpaque, + BuildableFromDrv +> Buildable; + +typedef std::vector Buildables; + +nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); + +} From 7a2b566dc8f0f94fdd6acbce90e47cd967f9f134 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 00:47:00 +0000 Subject: [PATCH 222/725] Move `StorePathWithOutputs` into its own header/file In the following commits it will become less prevalent. --- src/libcmd/installables.hh | 1 + src/libstore/derivations.cc | 8 ------- src/libstore/path-with-outputs.cc | 36 +++++++++++++++++++++++++++++++ src/libstore/path-with-outputs.hh | 17 +++++++++++++++ src/libstore/path.cc | 15 ------------- src/libstore/path.hh | 10 --------- src/libstore/store-api.cc | 7 ------ src/libstore/store-api.hh | 1 + 8 files changed, 55 insertions(+), 40 deletions(-) create mode 100644 src/libstore/path-with-outputs.cc create mode 100644 src/libstore/path-with-outputs.hh diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d31afd3d5..e5c6fe208 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "buildable.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe98182bb..f6defd98f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -590,14 +590,6 @@ std::map staticOutputHashes(Store& store, const Derivation& d } -std::string StorePathWithOutputs::to_string(const Store & store) const -{ - return outputs.empty() - ? store.printStorePath(path) - : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); -} - - bool wantOutput(const string & output, const std::set & wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc new file mode 100644 index 000000000..ba15df0a9 --- /dev/null +++ b/src/libstore/path-with-outputs.cc @@ -0,0 +1,36 @@ +#include "store-api.hh" + +namespace nix { + +std::string StorePathWithOutputs::to_string(const Store & store) const +{ + return outputs.empty() + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); +} + + +std::pair parsePathWithOutputs(std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? std::make_pair(s, std::set()) + : std::make_pair(((std::string_view) s).substr(0, n), + tokenizeString>(((std::string_view) s).substr(n + 1), ",")); +} + + +StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +{ + auto [path, outputs] = nix::parsePathWithOutputs(s); + return {parseStorePath(path), std::move(outputs)}; +} + + +StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +{ + auto [path2, outputs] = nix::parsePathWithOutputs(path); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; +} + +} diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh new file mode 100644 index 000000000..a9e3fc7c2 --- /dev/null +++ b/src/libstore/path-with-outputs.hh @@ -0,0 +1,17 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct StorePathWithOutputs +{ + StorePath path; + std::set outputs; + + std::string to_string(const Store & store) const; +}; + +std::pair parsePathWithOutputs(std::string_view s); + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index dc9dc3897..e642abcd5 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -std::pair parsePathWithOutputs(std::string_view s) -{ - size_t n = s.find("!"); - return n == s.npos - ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); -} - -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) -{ - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; -} - } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..06ba0663b 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -69,16 +69,6 @@ typedef std::map> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -struct StorePathWithOutputs -{ - StorePath path; - std::set outputs; - - std::string to_string(const Store & store) const; -}; - -std::pair parsePathWithOutputs(std::string_view s); - } namespace std { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5e321cedf..e3500872c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -53,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const -{ - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; -} - - /* Store paths have the following form: = /- diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5d19e8949..7adbe3b17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "realisation.hh" #include "path.hh" +#include "path-with-outputs.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 32f4454b9fa3ac30d58e738ece322eb19a0728ba Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 01:06:08 +0000 Subject: [PATCH 223/725] Move `StorePathWithOutput` utilities out of store class These are by no means part of the notion of a store, but rather are things that happen to use stores. (Or put another way, there's no way we'd make them virtual methods any time soon.) It's better to move them out of that too-big class then. Also, this helps us remove StorePathWithOutputs from the Store interface altogether next commit. --- src/libexpr/get-drvs.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/path-with-outputs.cc | 12 ++++++------ src/libstore/path-with-outputs.hh | 9 +++++++++ src/libstore/store-api.hh | 7 ------- src/nix-store/nix-store.cc | 4 ++-- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1a3990ea1..7793f26ff 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -19,7 +19,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); this->drvPath = store->printStorePath(drvPath); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f28ab6438..48706bff8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopBuildPaths: { std::vector drvs; for (auto & s : readStrings(from)) - drvs.push_back(store->parsePathWithOutputs(s)); + drvs.push_back(parsePathWithOutputs(*store, s)); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -861,7 +861,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryMissing: { std::vector targets; for (auto & s : readStrings(from)) - targets.push_back(store->parsePathWithOutputs(s)); + targets.push_back(parsePathWithOutputs(*store, s)); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index ba15df0a9..a898ad09c 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -20,17 +20,17 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs) { - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index a9e3fc7c2..0e34b5aa1 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,4 +14,13 @@ struct StorePathWithOutputs std::pair parsePathWithOutputs(std::string_view s); +class Store; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7adbe3b17..da7ac4460 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -262,11 +262,6 @@ public: PathSet printStorePathSet(const StorePathSet & path) const; - /* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ - StorePathWithOutputs parsePathWithOutputs(const string & s); - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ std::string showPaths(const StorePathSet & paths); @@ -290,8 +285,6 @@ public: result. */ StorePath followLinksToStorePath(std::string_view path) const; - StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; - /* Constructs a unique store path name. */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b684feccb..bfd1299fc 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -128,7 +128,7 @@ static void opRealise(Strings opFlags, Strings opArgs) std::vector paths; for (auto & i : opArgs) - paths.push_back(store->followLinksToStorePathWithOutputs(i)); + paths.push_back(followLinksToStorePathWithOutputs(*store, i)); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; @@ -873,7 +873,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths; for (auto & s : readStrings(in)) - paths.push_back(store->parsePathWithOutputs(s)); + paths.push_back(parsePathWithOutputs(*store, s)); getBuildSettings(); From 255d145ba7ac907d1cba8d088da556b591627756 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Mar 2021 03:50:41 +0000 Subject: [PATCH 224/725] Use `BuildableReq` for `buildPaths` and `ensurePath` This avoids an ambiguity where the `StorePathWithOutputs { drvPath, {} }` could mean "build `brvPath`" or "substitute `drvPath`" depending on context. It also brings the internals closer in line to the new CLI, by generalizing the `Buildable` type is used there and makes that distinction already. In doing so, relegate `StorePathWithOutputs` to being a type just for backwards compatibility (CLI and RPC). --- src/libcmd/installables.cc | 7 +-- src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 12 +++-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 3 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 16 +++--- src/libstore/build/local-derivation-goal.cc | 52 +++++++++++++------ src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 47 +++++++++++++++++ src/libstore/buildable.hh | 29 +++++++++-- src/libstore/daemon.cc | 21 +++++--- src/libstore/legacy-ssh-store.cc | 16 ++++-- src/libstore/misc.cc | 49 +++++++++--------- src/libstore/path-with-outputs.cc | 35 +++++++++++++ src/libstore/path-with-outputs.hh | 9 ++++ src/libstore/remote-store.cc | 57 +++++++++++++++++---- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 +-- src/libstore/store-api.hh | 6 +-- src/libstore/worker-protocol.hh | 22 ++++++++ src/nix-build/nix-build.cc | 4 +- src/nix-env/nix-env.cc | 30 ++++++----- src/nix-env/user-env.cc | 9 +++- src/nix-store/nix-store.cc | 11 ++-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 3 +- src/nix/flake.cc | 5 +- src/nix/profile.cc | 15 +++--- src/nix/run.cc | 2 +- 31 files changed, 364 insertions(+), 126 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ca416b9ee..b68c5f6a7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -679,19 +679,20 @@ Buildables build(ref store, Realise mode, Buildables buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { for (auto & b : i->toBuildables()) { std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path}); + pathsToBuild.push_back(bo); }, [&](BuildableFromDrv bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); - pathsToBuild.push_back({bfd.drvPath, outputNames}); + pathsToBuild.push_back( + BuildableReqFromDrv{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7793f26ff..f774e6493 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "eval-inline.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include #include diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d1afa768..24bc34b74 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); + drvs.push_back({ctx, {outputName}}); } } @@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context) if (!evalSettings.enableImportFromDerivation) throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", - store->printStorePath(drvs.begin()->path)); + store->printStorePath(drvs.begin()->drvPath)); /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed paths. */ diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5baaff3e9..20027e099 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index edc7b5efa..18e0fb57d 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,6 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" +#include "buildable.hh" #include @@ -42,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e7be517e..8680d0bce 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 686364440..d1973d78b 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,16 +6,20 @@ namespace nix { -void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & br : reqs) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](BuildableOpaque bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ef43c225..c245527c9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1190,6 +1190,26 @@ void LocalDerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } + +static StorePath pathPartOfReq(const BuildableReq & req) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) { + return bo.path; + }, + [&](BuildableReqFromDrv bfd) { + return bfd.drvPath; + }, + }, req); +} + + +bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +{ + return this->isAllowed(pathPartOfReq(req)); +} + + struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -1312,25 +1332,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + for (auto & req : paths) { + if (!goal.isAllowed(req)) + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); } next->buildPaths(paths, buildMode); for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); + auto p = std::get_if(&path); + if (!p) continue; + auto & bfd = *p; + auto outputs = next->queryDerivationOutputMap(bfd.drvPath); + for (auto & [outputName, outputPath] : outputs) + if (wantOutput(outputName, bfd.outputs)) + newPaths.insert(outputPath); } StorePathSet closure; @@ -1358,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1366,12 +1388,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); + std::vector allowed; + for (auto & req : targets) { + if (goal.isAllowed(req)) + allowed.emplace_back(req); else - unknown.insert(path.path); + unknown.insert(pathPartOfReq(req)); } next->queryMissing(allowed, willBuild, willSubstitute, diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 47b818a8b..edb93f84e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,6 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const BuildableReq & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 616b17e61..fef4cb0cb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->storePath}); + topPaths.push_back(BuildableOpaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 5cba45b1d..63ca1779e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,6 +11,7 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } +template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -30,4 +31,50 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) return res; } + +std::string BuildableOpaque::to_string(const Store & store) const { + return store.printStorePath(path); +} + +template<> +std::string BuildableReqFromDrv::to_string(const Store & store) const { + return store.printStorePath(drvPath) + + "!" + + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); +} + +std::string to_string(const Store & store, const BuildableReq & req) +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + req); +} + + +BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +{ + return {store.parseStorePath(s)}; +} + +template<> +BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + assert(n != s.npos); + auto drvPath = store.parseStorePath(s.substr(0, n)); + auto outputsS = s.substr(n + 1); + std::set outputs; + if (outputsS != "*") + outputs = tokenizeString>(outputsS); + return {drvPath, outputs}; +} + +BuildableReq parseBuildableReq(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? (BuildableReq) BuildableOpaque::parse(store, s) + : (BuildableReq) BuildableReqFromDrv::parse(store, s); +} + } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 6177237be..db78316bd 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "path.hh" +#include "path.hh" #include @@ -13,19 +14,37 @@ class Store; struct BuildableOpaque { StorePath path; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableOpaque parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +template +struct BuildableForFromDrv { StorePath drvPath; - std::map> outputs; + Outputs outputs; + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static BuildableForFromDrv parse(const Store & store, std::string_view); }; -typedef std::variant< +template +using BuildableFor = std::variant< BuildableOpaque, - BuildableFromDrv -> Buildable; + BuildableForFromDrv +>; + +typedef BuildableForFromDrv> BuildableReqFromDrv; +typedef BuildableFor> BuildableReq; + +std::string to_string(const Store & store, const BuildableReq &); + +BuildableReq parseBuildableReq(const Store & store, std::string_view); + +typedef BuildableForFromDrv>> BuildableFromDrv; +typedef BuildableFor>> Buildable; typedef std::vector Buildables; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48706bff8..6b527dcb2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,6 +2,7 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -259,6 +260,18 @@ static void writeValidPathInfo( } } +static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +{ + std::vector reqs; + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + reqs = worker_proto::read(store, from, Phantom> {}); + } else { + for (auto & s : readStrings(from)) + reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + } + return reqs; +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - std::vector drvs; - for (auto & s : readStrings(from)) - drvs.push_back(parsePathWithOutputs(*store, s)); + auto drvs = readBuildableReqs(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -859,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - std::vector targets; - for (auto & s : readStrings(from)) - targets.push_back(parsePathWithOutputs(*store, s)); + auto targets = readBuildableReqs(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a9f53bad9..1cb977be6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "serve-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" @@ -266,14 +267,23 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(*this)); + }, + [&](StorePath drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + }, sOrDrvPath); + } conn->to << ss; putBuildSettings(*conn); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f58816ad8..e702a4f9e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); + pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,24 +177,25 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); + pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); } } } }; - doPath = [&](const StorePathWithOutputs & path) { + doPath = [&](const BuildableReq & req) { { auto state(state_.lock()); - if (!state->done.insert(path.to_string(*this)).second) return; + if (!state->done.insert(to_string(*this, req)).second) return; } - if (path.path.isDerivation()) { - if (!isValidPath(path.path)) { + std::visit(overloaded { + [&](BuildableReqFromDrv bfd) { + if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bfd.drvPath); return; } @@ -202,52 +203,54 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; } - if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(StorePath(path.path), *drv); + auto drv = make_ref(derivationFromPath(bfd.drvPath)); + ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); } else - mustBuildDrv(path.path, *drv); + mustBuildDrv(bfd.drvPath, *drv); - } else { + }, + [&](BuildableOpaque bo) { - if (isValidPath(path.path)) return; + if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bo.path); return; } - auto info = infos.find(path.path); + auto info = infos.find(bo.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path.path); + state->willSubstitute.insert(bo.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); - } + pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + }, + }, req); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index a898ad09c..353286ac6 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,3 +1,4 @@ +#include "path-with-outputs.hh" #include "store-api.hh" namespace nix { @@ -10,6 +11,40 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } +BuildableReq StorePathWithOutputs::toBuildableReq() const +{ + if (!outputs.empty() || path.isDerivation()) + return BuildableReqFromDrv { path, outputs }; + else + return BuildableOpaque { path }; +} + + +std::vector toBuildableReqs(const std::vector ss) +{ + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + return reqs; +} + + +std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +{ + return std::visit(overloaded { + [&](BuildableOpaque bo) -> std::variant { + if (bo.path.isDerivation()) { + // drv path gets interpreted as "build", not "get drv file itself" + return bo.path; + } + return StorePathWithOutputs { bo.path }; + }, + [&](BuildableReqFromDrv bfd) -> std::variant { + return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + }, + }, p); +} + + std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0e34b5aa1..870cac08e 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,6 +1,9 @@ #pragma once +#include + #include "path.hh" +#include "buildable.hh" namespace nix { @@ -10,8 +13,14 @@ struct StorePathWithOutputs std::set outputs; std::string to_string(const Store & store) const; + + BuildableReq toBuildableReq() const; + + static std::variant tryFromBuildableReq(const BuildableReq &); }; +std::vector toBuildableReqs(const std::vector); + std::pair parsePathWithOutputs(std::string_view s); class Store; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ccf095dc2..de1c95ed6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "path-with-outputs.hh" #include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" @@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } + +BuildableReq read(const Store & store, Source & from, Phantom _) +{ + auto s = readString(from); + return parseBuildableReq(store, s); +} + +void write(const Store & store, Sink & out, const BuildableReq & req) +{ + out << to_string(store, req); +} + + Realisation read(const Store & store, Source & from, Phantom _) { std::string rawInput = readString(from); @@ -58,8 +72,12 @@ Realisation read(const Store & store, Source & from, Phantom _) "remote-protocol" ); } + void write(const Store & store, Sink & out, const Realisation & realisation) -{ out << realisation.toJSON().dump(); } +{ + out << realisation.toJSON().dump(); +} + DrvOutput read(const Store & store, Source & from, Phantom _) { @@ -652,16 +670,36 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } +static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +{ + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + worker_proto::write(store, conn->to, reqs); + } else { + Strings ss; + for (auto & p : reqs) { + auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(store)); + }, + [&](StorePath drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn->daemonVersion), + GET_PROTOCOL_MINOR(conn->daemonVersion)); + }, + }, sOrDrvPath); + } + conn->to << ss; + } +} -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -800,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -811,10 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeBuildableReqs(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b3a9910a3..20d366038 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e3500872c..8b60bdc62 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back({path}); + paths2.push_back(BuildableOpaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index da7ac4460..59d0983df 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "buildable.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index be071dd78..0255726ac 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,9 +86,11 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, BuildableReq); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); #define X_ template @@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); +template +std::vector read(const Store & store, Source & from, Phantom> _) +{ + std::vector resSet; + auto size = readNum(from); + while (size--) { + resSet.push_back(read(store, from, Phantom {})); + } + return resSet; +} + +template +void write(const Store & store, Sink & out, const std::vector & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + template std::set read(const Store & store, Source & from, Phantom> _) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 65b85b304..6f8a61261 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -12,6 +12,7 @@ #include "affinity.hh" #include "util.hh" #include "shared.hh" +#include "path-with-outputs.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" @@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector & paths) { + auto buildPaths = [&](const std::vector & paths0) { + auto paths = toBuildableReqs(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0f10a4cbb..af1c69b87 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -6,6 +6,7 @@ #include "globals.hh" #include "names.hh" #include "profiles.hh" +#include "path-with-outputs.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back({state.store->parseStorePath(drvPath)}); + targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); else - targets.push_back({state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - if (drv.queryDrvPath() != "") { - std::vector paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; - printMissing(globals.state->store, paths); - if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } else { - printMissing(globals.state->store, - {{globals.state->store->parseStorePath(drv.queryOutPath())}}); - if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); - } + std::vector paths { + (drv.queryDrvPath() != "") + ? (BuildableReq) (BuildableReqFromDrv { + globals.state->store->parseStorePath(drv.queryDrvPath()) + }) + : (BuildableReq) (BuildableOpaque { + globals.state->store->parseStorePath(drv.queryOutPath()) + }), + }; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); debug(format("switching to new user environment")); Path generation = createGeneration( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 168ac492b..0ccf960fb 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" @@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); - state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(drvsToBuild), + state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; @@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toBuildableReqs(topLevelDrvs), + state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index bfd1299fc..21c1e547b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -10,6 +10,7 @@ #include "worker-protocol.hh" #include "graphml.hh" #include "legacy.hh" +#include "path-with-outputs.hh" #include #include @@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path}); + if (build) store->buildPaths({path.toBuildableReq()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -132,7 +133,9 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->queryMissing( + toBuildableReqs(paths), + willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { std::vector paths2; @@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(paths, buildMode); + store->buildPaths(toBuildableReqs(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -879,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(paths); + store->buildPaths(toBuildableReqs(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 48f4eb6e3..e86fbb3f7 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(app.context); + store->buildPaths(toBuildableReqs(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({{drvPath}}); + store->buildPaths({ BuildableReqFromDrv { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d0b140570..616e2073e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" @@ -159,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({{shellDrvPath}}); + store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a2b6c0303..9d6d22a43 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" +#include "path-with-outputs.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -292,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -461,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back({drvPath}); + drvPaths.push_back(BuildableReqFromDrv{drvPath}); } } } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4d275f577..b96e71844 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -260,12 +260,15 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile std::visit(overloaded { [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path, {}}); + pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, [&](BuildableFromDrv bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back({bfd.drvPath, {output.first}}); + pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -388,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -423,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index ec9388234..2e9bb41cc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(app.context); + state->store->buildPaths(toBuildableReqs(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 4fe41c6db390c0295d20f6365ebedaec8ec79e1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:15:25 -0400 Subject: [PATCH 225/725] No templates for `Buildable` and `BuildableReq` --- src/libstore/buildable.cc | 3 --- src/libstore/buildable.hh | 30 +++++++++++++++++------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 63ca1779e..7892b94e4 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,6 @@ nlohmann::json BuildableOpaque::toJSON(ref store) const { return res; } -template<> nlohmann::json BuildableFromDrv::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -36,7 +35,6 @@ std::string BuildableOpaque::to_string(const Store & store) const { return store.printStorePath(path); } -template<> std::string BuildableReqFromDrv::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" @@ -56,7 +54,6 @@ BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) return {store.parseStorePath(s)}; } -template<> BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index db78316bd..54e627271 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -20,31 +20,35 @@ struct BuildableOpaque { static BuildableOpaque parse(const Store & store, std::string_view); }; -template -struct BuildableForFromDrv { +struct BuildableReqFromDrv { StorePath drvPath; - Outputs outputs; + std::set outputs; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableForFromDrv parse(const Store & store, std::string_view); + static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -template -using BuildableFor = std::variant< +using BuildableReq = std::variant< BuildableOpaque, - BuildableForFromDrv + BuildableReqFromDrv >; -typedef BuildableForFromDrv> BuildableReqFromDrv; -typedef BuildableFor> BuildableReq; - std::string to_string(const Store & store, const BuildableReq &); BuildableReq parseBuildableReq(const Store & store, std::string_view); -typedef BuildableForFromDrv>> BuildableFromDrv; -typedef BuildableFor>> Buildable; +struct BuildableFromDrv { + StorePath drvPath; + std::map> outputs; + + nlohmann::json toJSON(ref store) const; + static BuildableFromDrv parse(const Store & store, std::string_view); +}; + +using Buildable = std::variant< + BuildableOpaque, + BuildableFromDrv +>; typedef std::vector Buildables; From 9dfb97c987d8b9d6a3d15f016e40f22f91deb764 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:24:42 -0400 Subject: [PATCH 226/725] "newtype" BuildableReq This makes for better types errors and allows us to give it methods. --- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 4 ++-- src/libstore/buildable.cc | 6 +++--- src/libstore/buildable.hh | 14 +++++++++++--- src/libstore/misc.cc | 4 ++-- src/libstore/path-with-outputs.cc | 2 +- src/libstore/remote-store.cc | 4 ++-- 8 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8680d0bce..8396abbcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - to_string(worker.store, BuildableReqFromDrv { drvPath, wantedOutputs })); + BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - to_string(worker.store, BuildableReqFromDrv { drvPath, drv.outputNames() })); + BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index d1973d78b..fc6294545 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -19,7 +19,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMo [&](BuildableOpaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, - }, br); + }, br.raw()); } worker.run(goals); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c245527c9..6cc384719 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1200,7 +1200,7 @@ static StorePath pathPartOfReq(const BuildableReq & req) [&](BuildableReqFromDrv bfd) { return bfd.drvPath; }, - }, req); + }, req.raw()); } @@ -1340,7 +1340,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo for (auto & req : paths) { if (!goal.isAllowed(req)) - throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", to_string(*next, req)); + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } next->buildPaths(paths, buildMode); diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 7892b94e4..31fef2faa 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -41,11 +41,11 @@ std::string BuildableReqFromDrv::to_string(const Store & store) const { + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string to_string(const Store & store, const BuildableReq & req) +std::string BuildableReq::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, - req); + this->raw()); } @@ -66,7 +66,7 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq parseBuildableReq(const Store & store, std::string_view s) +BuildableReq BuildableReq::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 54e627271..8317f3995 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -28,14 +28,22 @@ struct BuildableReqFromDrv { static BuildableReqFromDrv parse(const Store & store, std::string_view); }; -using BuildableReq = std::variant< +using _BuildableReqRaw = std::variant< BuildableOpaque, BuildableReqFromDrv >; -std::string to_string(const Store & store, const BuildableReq &); +struct BuildableReq : _BuildableReqRaw { + using Raw = _BuildableReqRaw; + using Raw::Raw; -BuildableReq parseBuildableReq(const Store & store, std::string_view); + inline const Raw & raw() const { + return static_cast(*this); + } + + std::string to_string(const Store & store) const; + static BuildableReq parse(const Store & store, std::string_view); +}; struct BuildableFromDrv { StorePath drvPath; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e702a4f9e..abfae1502 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -187,7 +187,7 @@ void Store::queryMissing(const std::vector & targets, { auto state(state_.lock()); - if (!state->done.insert(to_string(*this, req)).second) return; + if (!state->done.insert(req.to_string(*this)).second) return; } std::visit(overloaded { @@ -250,7 +250,7 @@ void Store::queryMissing(const std::vector & targets, for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); }, - }, req); + }, req.raw()); }; for (auto & path : targets) diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 353286ac6..2898b8d4f 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -41,7 +41,7 @@ std::variant StorePathWithOutputs::tryFromBuild [&](BuildableReqFromDrv bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, - }, p); + }, p.raw()); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de1c95ed6..cb6402213 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -55,12 +55,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) BuildableReq read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return parseBuildableReq(store, s); + return BuildableReq::parse(store, s); } void write(const Store & store, Sink & out, const BuildableReq & req) { - out << to_string(store, req); + out << req.to_string(store); } From 9b805d36ac70545fc4c0d863e21e0c2e5f2518a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 09:48:18 -0400 Subject: [PATCH 227/725] Rename Buildable --- src/libcmd/command.cc | 6 +-- src/libcmd/command.hh | 4 +- src/libcmd/installables.cc | 44 ++++++++++----------- src/libcmd/installables.hh | 6 +-- src/libexpr/primops.cc | 6 +-- src/libmain/shared.cc | 2 +- src/libmain/shared.hh | 2 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/build/entry-points.cc | 6 +-- src/libstore/build/local-derivation-goal.cc | 16 ++++---- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/worker.cc | 6 +-- src/libstore/buildable.cc | 24 +++++------ src/libstore/buildable.hh | 37 +++++++++-------- src/libstore/daemon.cc | 12 +++--- src/libstore/legacy-ssh-store.cc | 4 +- src/libstore/misc.cc | 16 ++++---- src/libstore/path-with-outputs.cc | 18 ++++----- src/libstore/path-with-outputs.hh | 6 +-- src/libstore/remote-store.cc | 18 ++++----- src/libstore/remote-store.hh | 4 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/worker-protocol.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 12 +++--- src/nix-env/user-env.cc | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/build.cc | 6 +-- src/nix/bundle.cc | 4 +- src/nix/develop.cc | 8 ++-- src/nix/flake.cc | 4 +- src/nix/log.cc | 6 +-- src/nix/profile.cc | 14 +++---- src/nix/run.cc | 2 +- 35 files changed, 165 insertions(+), 162 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d29954f67..dc1fbc43f 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -162,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const Buildables & buildables) +void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) { if (!profile) return; @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { result.push_back(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index e66c697eb..9e18c6e51 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, @@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const Buildables & buildables); + void updateProfile(const DerivedPathsWithHints & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b68c5f6a7..f091ac186 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -285,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -Buildable Installable::toBuildable() +DerivedPathWithHints Installable::toDerivedPathWithHints() { - auto buildables = toBuildables(); + auto buildables = toDerivedPathsWithHints(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -321,7 +321,7 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - Buildables toBuildables() override + DerivedPathsWithHints toDerivedPathsWithHints() override { if (storePath.isDerivation()) { std::map> outputs; @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - BuildableFromDrv { + DerivedPathWithHintsBuilt { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - BuildableOpaque { + DerivedPathOpaque { .path = storePath, } }; @@ -349,9 +349,9 @@ struct InstallableStorePath : Installable } }; -Buildables InstallableValue::toBuildables() +DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() { - Buildables res; + DerivedPathsWithHints res; std::map>> drvsToOutputs; @@ -364,7 +364,7 @@ Buildables InstallableValue::toBuildables() } for (auto & i : drvsToOutputs) - res.push_back(BuildableFromDrv { i.first, i.second }); + res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); return res; } @@ -671,28 +671,28 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -Buildables build(ref store, Realise mode, +DerivedPathsWithHints build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - Buildables buildables; + DerivedPathsWithHints buildables; - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toBuildables()) { + for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( - BuildableReqFromDrv{bfd.drvPath, outputNames}); + DerivedPath::Built{bfd.drvPath, outputNames}); }, }, b); buildables.push_back(std::move(b)); @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { res.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -751,8 +751,8 @@ std::set toRealisedPaths( settings.readOnlyMode = true; for (auto & i : installables) - for (auto & b : i->toBuildables()) - if (auto bfd = std::get_if(&b)) + for (auto & b : i->toDerivedPathsWithHints()) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -787,9 +787,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) + for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,7 +798,7 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { drvPaths.insert(bfd.drvPath); }, }, b); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e5c6fe208..0bc932b52 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -29,9 +29,9 @@ struct Installable virtual std::string what() = 0; - virtual Buildables toBuildables() = 0; + virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0; - Buildable toBuildable(); + DerivedPathWithHints toDerivedPathWithHints(); App toApp(EvalState & state); @@ -74,7 +74,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - Buildables toBuildables() override; + DerivedPathsWithHints toDerivedPathsWithHints() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 24bc34b74..428adf4c2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -56,8 +56,8 @@ void EvalState::realiseContext(const PathSet & context) /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - std::vector buildReqs; - for (auto & d : drvs) buildReqs.emplace_back(BuildableReq { d }); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(buildReqs); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 20027e099..09af57871 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 18e0fb57d..9cb9e6da2 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -43,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8396abbcd..3ce538f77 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -73,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - BuildableReqFromDrv { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +94,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - BuildableReqFromDrv { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index fc6294545..732d4785d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,17 +6,17 @@ namespace nix { -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; for (auto & br : reqs) { std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); }, }, br.raw()); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6cc384719..7c1402918 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1191,20 +1191,20 @@ void LocalDerivationGoal::writeStructuredAttrs() } -static StorePath pathPartOfReq(const BuildableReq & req) +static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { return bo.path; }, - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { return bfd.drvPath; }, }, req.raw()); } -bool LocalDerivationGoal::isAllowed(const BuildableReq & req) +bool LocalDerivationGoal::isAllowed(const DerivedPath & req) { return this->isAllowed(pathPartOfReq(req)); } @@ -1332,7 +1332,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // an allowed derivation { throw Error("queryRealisation"); } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1346,7 +1346,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo next->buildPaths(paths, buildMode); for (auto & path : paths) { - auto p = std::get_if(&path); + auto p = std::get_if(&path); if (!p) continue; auto & bfd = *p; auto outputs = next->queryDerivationOutputMap(bfd.drvPath); @@ -1380,7 +1380,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1388,7 +1388,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; + std::vector allowed; for (auto & req : targets) { if (goal.isAllowed(req)) allowed.emplace_back(req); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index edb93f84e..d30be2351 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -116,7 +116,7 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } - bool isAllowed(const BuildableReq & req); + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index fef4cb0cb..6c04d3ed3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -226,14 +226,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableReqFromDrv{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(BuildableOpaque{goal->storePath}); + topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index 31fef2faa..a8c0c70b1 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -5,13 +5,13 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { +nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { nlohmann::json res; res["path"] = store->printStorePath(path); return res; } -nlohmann::json BuildableFromDrv::toJSON(ref store) const { +nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -20,9 +20,9 @@ nlohmann::json BuildableFromDrv::toJSON(ref store) const { return res; } -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store) { auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { + for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); }, buildable); @@ -31,17 +31,17 @@ nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) } -std::string BuildableOpaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } -std::string BuildableReqFromDrv::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); } -std::string BuildableReq::to_string(const Store & store) const +std::string DerivedPath::to_string(const Store & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, @@ -49,12 +49,12 @@ std::string BuildableReq::to_string(const Store & store) const } -BuildableOpaque BuildableOpaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) { return {store.parseStorePath(s)}; } -BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); assert(n != s.npos); @@ -66,12 +66,12 @@ BuildableReqFromDrv BuildableReqFromDrv::parse(const Store & store, std::string_ return {drvPath, outputs}; } -BuildableReq BuildableReq::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? (BuildableReq) BuildableOpaque::parse(store, s) - : (BuildableReq) BuildableReqFromDrv::parse(store, s); + ? (DerivedPath) DerivedPath::Opaque::parse(store, s) + : (DerivedPath) DerivedPath::Built::parse(store, s); } } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 8317f3995..0a0cf8105 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -12,54 +12,57 @@ namespace nix { class Store; -struct BuildableOpaque { +struct DerivedPathOpaque { StorePath path; nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; - static BuildableOpaque parse(const Store & store, std::string_view); + static DerivedPathOpaque parse(const Store & store, std::string_view); }; -struct BuildableReqFromDrv { +struct DerivedPathBuilt { StorePath drvPath; std::set outputs; std::string to_string(const Store & store) const; - static BuildableReqFromDrv parse(const Store & store, std::string_view); + static DerivedPathBuilt parse(const Store & store, std::string_view); }; -using _BuildableReqRaw = std::variant< - BuildableOpaque, - BuildableReqFromDrv +using _DerivedPathRaw = std::variant< + DerivedPathOpaque, + DerivedPathBuilt >; -struct BuildableReq : _BuildableReqRaw { - using Raw = _BuildableReqRaw; +struct DerivedPath : _DerivedPathRaw { + using Raw = _DerivedPathRaw; using Raw::Raw; + using Opaque = DerivedPathOpaque; + using Built = DerivedPathBuilt; + inline const Raw & raw() const { return static_cast(*this); } std::string to_string(const Store & store) const; - static BuildableReq parse(const Store & store, std::string_view); + static DerivedPath parse(const Store & store, std::string_view); }; -struct BuildableFromDrv { +struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; nlohmann::json toJSON(ref store) const; - static BuildableFromDrv parse(const Store & store, std::string_view); + static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using Buildable = std::variant< - BuildableOpaque, - BuildableFromDrv +using DerivedPathWithHints = std::variant< + DerivedPath::Opaque, + DerivedPathWithHintsBuilt >; -typedef std::vector Buildables; +typedef std::vector DerivedPathsWithHints; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); +nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6b527dcb2..affd60472 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,14 +260,14 @@ static void writeValidPathInfo( } } -static std::vector readBuildableReqs(Store & store, unsigned int clientVersion, Source & from) +static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { - std::vector reqs; + std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - reqs = worker_proto::read(store, from, Phantom> {}); + reqs = worker_proto::read(store, from, Phantom> {}); } else { for (auto & s : readStrings(from)) - reqs.push_back(parsePathWithOutputs(store, s).toBuildableReq()); + reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); } return reqs; } @@ -506,7 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - auto drvs = readBuildableReqs(*store, clientVersion, from); + auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -870,7 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - auto targets = readBuildableReqs(*store, clientVersion, from); + auto targets = readDerivedPaths(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1cb977be6..edaf75136 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,14 +267,14 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(*this)); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index abfae1502..a99a2fc78 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -117,7 +117,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +145,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +154,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, BuildableReqFromDrv { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,13 +177,13 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, BuildableOpaque { path } )); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } )); } } } }; - doPath = [&](const BuildableReq & req) { + doPath = [&](const DerivedPath & req) { { auto state(state_.lock()); @@ -191,7 +191,7 @@ void Store::queryMissing(const std::vector & targets, } std::visit(overloaded { - [&](BuildableReqFromDrv bfd) { + [&](DerivedPath::Built bfd) { if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); @@ -224,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, mustBuildDrv(bfd.drvPath, *drv); }, - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { if (isValidPath(bo.path)) return; @@ -248,7 +248,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, BuildableOpaque { ref })); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); }; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 2898b8d4f..865d64cf2 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -11,34 +11,34 @@ std::string StorePathWithOutputs::to_string(const Store & store) const } -BuildableReq StorePathWithOutputs::toBuildableReq() const +DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty() || path.isDerivation()) - return BuildableReqFromDrv { path, outputs }; + return DerivedPath::Built { path, outputs }; else - return BuildableOpaque { path }; + return DerivedPath::Opaque { path }; } -std::vector toBuildableReqs(const std::vector ss) +std::vector toDerivedPaths(const std::vector ss) { - std::vector reqs; - for (auto & s : ss) reqs.push_back(s.toBuildableReq()); + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); return reqs; } -std::variant StorePathWithOutputs::tryFromBuildableReq(const BuildableReq & p) +std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](BuildableOpaque bo) -> std::variant { + [&](DerivedPath::Opaque bo) -> std::variant { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](BuildableReqFromDrv bfd) -> std::variant { + [&](DerivedPath::Built bfd) -> std::variant { return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; }, }, p.raw()); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 870cac08e..749348398 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -14,12 +14,12 @@ struct StorePathWithOutputs std::string to_string(const Store & store) const; - BuildableReq toBuildableReq() const; + DerivedPath toDerivedPath() const; - static std::variant tryFromBuildableReq(const BuildableReq &); + static std::variant tryFromDerivedPath(const DerivedPath &); }; -std::vector toBuildableReqs(const std::vector); +std::vector toDerivedPaths(const std::vector); std::pair parsePathWithOutputs(std::string_view s); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cb6402213..761b4a087 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -52,13 +52,13 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) } -BuildableReq read(const Store & store, Source & from, Phantom _) +DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return BuildableReq::parse(store, s); + return DerivedPath::parse(store, s); } -void write(const Store & store, Sink & out, const BuildableReq & req) +void write(const Store & store, Sink & out, const DerivedPath & req) { out << req.to_string(store); } @@ -670,14 +670,14 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & return {Realisation{.id = id, .outPath = *outPaths.begin()}}; } -static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { worker_proto::write(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromBuildableReq(p); + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); std::visit(overloaded { [&](StorePathWithOutputs s) { ss.push_back(s.to_string(store)); @@ -694,12 +694,12 @@ static void writeBuildableReqs(RemoteStore & store, ConnectionHandle & conn, con } } -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeBuildableReqs(*this, conn, drvPaths); + writeDerivedPaths(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -838,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -849,7 +849,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - writeBuildableReqs(*this, conn, targets); + writeDerivedPaths(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 20d366038..6cf76a46d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8b60bdc62..93fcb068f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -529,10 +529,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(BuildableOpaque{path}); + paths2.push_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -540,8 +540,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back(BuildableOpaque{p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 59d0983df..483f3c5fa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -494,7 +494,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -656,7 +656,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 0255726ac..001ed25e3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -86,7 +86,7 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, BuildableReq); +MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6f8a61261..d46bc1f2b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -323,7 +323,7 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); auto buildPaths = [&](const std::vector & paths0) { - auto paths = toBuildableReqs(paths0); + auto paths = toDerivedPaths(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index af1c69b87..e04954d45 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -419,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back(BuildableReqFromDrv{state.store->parseStorePath(drvPath)}); + targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)}); else - targets.push_back(BuildableOpaque{state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -694,12 +694,12 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - std::vector paths { + std::vector paths { (drv.queryDrvPath() != "") - ? (BuildableReq) (BuildableReqFromDrv { + ? (DerivedPath) (DerivedPath::Built { globals.state->store->parseStorePath(drv.queryDrvPath()) }) - : (BuildableReq) (BuildableOpaque { + : (DerivedPath) (DerivedPath::Opaque { globals.state->store->parseStorePath(drv.queryOutPath()) }), }; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 0ccf960fb..5ceb2ae67 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -43,7 +43,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug(format("building user environment dependencies")); state.store->buildPaths( - toBuildableReqs(drvsToBuild), + toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ @@ -140,7 +140,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); state.store->buildPaths( - toBuildableReqs(topLevelDrvs), + toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 21c1e547b..b327793e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -63,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path.toBuildableReq()}); + if (build) store->buildPaths({path.toDerivedPath()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -134,7 +134,7 @@ static void opRealise(Strings opFlags, Strings opArgs) uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing( - toBuildableReqs(paths), + toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { @@ -151,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(toBuildableReqs(paths), buildMode); + store->buildPaths(toDerivedPaths(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -882,7 +882,7 @@ static void opServe(Strings opFlags, Strings opArgs) try { MonitorFdHup monitor(in.fd); - store->buildPaths(toBuildableReqs(paths)); + store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { assert(e.status); diff --git a/src/nix/build.cc b/src/nix/build.cc index 724ce9d79..0529ed382 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -80,7 +80,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile updateProfile(buildables); - if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); } }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e86fbb3f7..53dccc63a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -70,7 +70,7 @@ struct CmdBundle : InstallableCommand auto evalState = getEvalState(); auto app = installable->toApp(*evalState); - store->buildPaths(toBuildableReqs(app.context)); + store->buildPaths(toDerivedPaths(app.context)); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +110,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({ BuildableReqFromDrv { drvPath } }); + store->buildPaths({ DerivedPath::Built { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 616e2073e..cae6ded40 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -160,7 +160,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({BuildableReqFromDrv{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}); for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { auto & [_1, optPath] = outputAndOptPath; @@ -265,7 +265,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toBuildable(); + auto buildable = installable->toDerivedPathWithHints(); auto doRedirect = [&](const StorePath & path) { auto from = store->printStorePath(path); @@ -277,10 +277,10 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const BuildableOpaque & bo) { + [&](const DerivedPathOpaque & bo) { doRedirect(bo.path); }, - [&](const BuildableFromDrv & bfd) { + [&](const DerivedPathWithHintsBuilt & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9d6d22a43..62a413e27 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -293,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -462,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back(BuildableReqFromDrv{drvPath}); + drvPaths.push_back(DerivedPath::Built{drvPath}); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index 67d3742d6..5010e3326 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,15 +30,15 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuildable(); + auto b = installable->toDerivedPathWithHints(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { return sub->getBuildLog(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { return sub->getBuildLog(bfd.drvPath); }, }, b); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b96e71844..ad824dd70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -259,16 +259,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPathOpaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPathWithHintsBuilt bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back(BuildableReqFromDrv{bfd.drvPath, {output.first}}); + pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, @@ -391,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -426,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back(BuildableReqFromDrv{drv.drvPath, {"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME } } diff --git a/src/nix/run.cc b/src/nix/run.cc index 2e9bb41cc..ba60e57d8 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,7 +182,7 @@ struct CmdRun : InstallableCommand, RunCommon auto app = installable->toApp(*state); - state->store->buildPaths(toBuildableReqs(app.context)); + state->store->buildPaths(toDerivedPaths(app.context)); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); From 179582872de60863fcabcf471f98930a25fd6df3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:05:21 -0400 Subject: [PATCH 228/725] Make `DerivedPathWithHints` a newtype This allows us to namespace its constructors under it. --- src/libcmd/command.cc | 6 +++--- src/libcmd/installables.cc | 26 +++++++++++++------------- src/libstore/buildable.cc | 4 ++-- src/libstore/buildable.hh | 15 ++++++++++++++- src/nix/build.cc | 6 +++--- src/nix/develop.cc | 6 +++--- src/nix/log.cc | 6 +++--- src/nix/profile.cc | 6 +++--- 8 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index dc1fbc43f..9da470c15 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -170,10 +170,10 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { result.push_back(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { for (auto & output : bfd.outputs) { /* Output path should be known because we just tried to build it. */ @@ -181,7 +181,7 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables) result.push_back(*output.second); } }, - }, buildable); + }, buildable.raw()); } if (result.size() != 1) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f091ac186..5d3026c1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -329,14 +329,14 @@ struct InstallableStorePath : Installable for (auto & [name, output] : drv.outputsAndOptPaths(*store)) outputs.emplace(name, output.second); return { - DerivedPathWithHintsBuilt { + DerivedPathWithHints::Built { .drvPath = storePath, .outputs = std::move(outputs) } }; } else { return { - DerivedPathOpaque { + DerivedPathWithHints::Opaque { .path = storePath, } }; @@ -364,7 +364,7 @@ DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints() } for (auto & i : drvsToOutputs) - res.push_back(DerivedPathWithHintsBuilt { i.first, i.second }); + res.push_back(DerivedPathWithHints::Built { i.first, i.second }); return res; } @@ -684,17 +684,17 @@ DerivedPathsWithHints build(ref store, Realise mode, for (auto & i : installables) { for (auto & b : i->toDerivedPathsWithHints()) { std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { StringSet outputNames; for (auto & output : bfd.outputs) outputNames.insert(output.first); pathsToBuild.push_back( DerivedPath::Built{bfd.drvPath, outputNames}); }, - }, b); + }, b.raw()); buildables.push_back(std::move(b)); } } @@ -717,10 +717,10 @@ std::set toRealisedPaths( if (operateOn == OperateOn::Output) { for (auto & b : build(store, mode, installables)) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { res.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto drv = store->readDerivation(bfd.drvPath); auto outputHashes = staticOutputHashes(*store, drv); for (auto & output : bfd.outputs) { @@ -745,14 +745,14 @@ std::set toRealisedPaths( } } }, - }, b); + }, b.raw()); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) - if (auto bfd = std::get_if(&b)) + if (auto bfd = std::get_if(&b)) res.insert(bfd->drvPath); } @@ -789,7 +789,7 @@ StorePathSet toDerivations(ref store, for (auto & i : installables) for (auto & b : i->toDerivedPathsWithHints()) std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -798,10 +798,10 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { drvPaths.insert(bfd.drvPath); }, - }, b); + }, b.raw()); return drvPaths; } diff --git a/src/libstore/buildable.cc b/src/libstore/buildable.cc index a8c0c70b1..eee38ba10 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/buildable.cc @@ -11,7 +11,7 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { return res; } -nlohmann::json DerivedPathWithHintsBuilt::toJSON(ref store) const { +nlohmann::json DerivedPathWithHints::Built::toJSON(ref store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); for (const auto& [output, path] : outputs) { @@ -25,7 +25,7 @@ nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildab for (const DerivedPathWithHints & buildable : buildables) { std::visit([&res, store](const auto & buildable) { res.push_back(buildable.toJSON(store)); - }, buildable); + }, buildable.raw()); } return res; } diff --git a/src/libstore/buildable.hh b/src/libstore/buildable.hh index 0a0cf8105..ce5ae5fc0 100644 --- a/src/libstore/buildable.hh +++ b/src/libstore/buildable.hh @@ -56,11 +56,24 @@ struct DerivedPathWithHintsBuilt { static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view); }; -using DerivedPathWithHints = std::variant< +using _DerivedPathWithHintsRaw = std::variant< DerivedPath::Opaque, DerivedPathWithHintsBuilt >; +struct DerivedPathWithHints : _DerivedPathWithHintsRaw { + using Raw = _DerivedPathWithHintsRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = DerivedPathWithHintsBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + +}; + typedef std::vector DerivedPathsWithHints; nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref store); diff --git a/src/nix/build.cc b/src/nix/build.cc index 0529ed382..03159b6cc 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -61,12 +61,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); for (auto & output : builtOutputs) { std::string symlink = outLink; @@ -75,7 +75,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildable); + }, buildable.raw()); } updateProfile(buildables); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index cae6ded40..7cc7b85be 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -277,14 +277,14 @@ struct Common : InstallableCommand, MixProfile } }; std::visit(overloaded { - [&](const DerivedPathOpaque & bo) { + [&](const DerivedPathWithHints::Opaque & bo) { doRedirect(bo.path); }, - [&](const DerivedPathWithHintsBuilt & bfd) { + [&](const DerivedPathWithHints::Built & bfd) { for (auto & [outputName, path] : bfd.outputs) if (path) doRedirect(*path); }, - }, buildable); + }, buildable.raw()); } return rewriteStrings(script, rewrites); diff --git a/src/nix/log.cc b/src/nix/log.cc index 5010e3326..638bb5073 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -35,13 +35,13 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, - }, b); + }, b.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ad824dd70..667904cd2 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -259,11 +259,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](DerivedPathOpaque bo) { + [&](DerivedPathWithHints::Opaque bo) { pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](DerivedPathWithHintsBuilt bfd) { + [&](DerivedPathWithHints::Built bfd) { // TODO: Why are we querying if we know the output // names already? Is it just to figure out what the // default one is? @@ -272,7 +272,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.storePaths.insert(output.second); } }, - }, buildable); + }, buildable.raw()); manifest.elements.emplace_back(std::move(element)); } From d8fa7517fad4272e20ff9b9b740c91158bc685e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:33:28 -0400 Subject: [PATCH 229/725] buildable.{cc,hh} -> derived-path.{cc,hh} --- src/libcmd/installables.hh | 2 +- src/libmain/shared.hh | 2 +- src/libstore/{buildable.cc => derived-path.cc} | 2 +- src/libstore/{buildable.hh => derived-path.hh} | 0 src/libstore/path-with-outputs.hh | 2 +- src/libstore/store-api.hh | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/libstore/{buildable.cc => derived-path.cc} (98%) rename src/libstore/{buildable.hh => derived-path.hh} (100%) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 0bc932b52..403403c07 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "path.hh" #include "path-with-outputs.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "eval.hh" #include "flake/flake.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9cb9e6da2..05277d90a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,7 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include diff --git a/src/libstore/buildable.cc b/src/libstore/derived-path.cc similarity index 98% rename from src/libstore/buildable.cc rename to src/libstore/derived-path.cc index eee38ba10..13833c58e 100644 --- a/src/libstore/buildable.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,4 @@ -#include "buildable.hh" +#include "derived-path.hh" #include "store-api.hh" #include diff --git a/src/libstore/buildable.hh b/src/libstore/derived-path.hh similarity index 100% rename from src/libstore/buildable.hh rename to src/libstore/derived-path.hh diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 749348398..4c4023dcb 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -3,7 +3,7 @@ #include #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 483f3c5fa..f66298991 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,7 +2,7 @@ #include "realisation.hh" #include "path.hh" -#include "buildable.hh" +#include "derived-path.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" From 125a824228dbac0bb82023953f45318ea93e7ffa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Apr 2021 10:56:48 -0400 Subject: [PATCH 230/725] Document the derived path types. --- src/libstore/derived-path.hh | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index ce5ae5fc0..7a2fe59de 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -2,7 +2,6 @@ #include "util.hh" #include "path.hh" -#include "path.hh" #include @@ -12,6 +11,13 @@ namespace nix { class Store; +/** + * An opaque derived path. + * + * Opaque derived paths are just store paths, and fully evaluated. They + * cannot be simplified further. Since they are opaque, they cannot be + * built, but they can fetched. + */ struct DerivedPathOpaque { StorePath path; @@ -20,6 +26,18 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const Store & store, std::string_view); }; +/** + * A derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ struct DerivedPathBuilt { StorePath drvPath; std::set outputs; @@ -33,6 +51,16 @@ using _DerivedPathRaw = std::variant< DerivedPathBuilt >; +/** + * A "derived path" is a very simple sort of expression that evaluates + * to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; using Raw::Raw; @@ -48,6 +76,11 @@ struct DerivedPath : _DerivedPathRaw { static DerivedPath parse(const Store & store, std::string_view); }; +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'DerivedPathWithHints' for more an explanation. + */ struct DerivedPathWithHintsBuilt { StorePath drvPath; std::map> outputs; @@ -61,6 +94,21 @@ using _DerivedPathWithHintsRaw = std::variant< DerivedPathWithHintsBuilt >; +/** + * A derived path with hints in the form of optional concrete output paths in the built case. + * + * This type is currently just used by the CLI. The paths are filled in + * during evaluation for derivations that know what paths they will + * produce in advanced, i.e. input-addressed or fixed-output content + * addressed derivations. + * + * That isn't very good, because it puts floating content-addressed + * derivations "at a disadvantage". It would be better to never rely on + * the output path of unbuilt derivations, and exclusively use the + * realizations types to work with built derivations' concrete output + * paths. + */ +// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd. struct DerivedPathWithHints : _DerivedPathWithHintsRaw { using Raw = _DerivedPathWithHintsRaw; using Raw::Raw; From 9f28dd97ae6afc68f0574a251325336c12d60c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gohla?= Date: Mon, 5 Apr 2021 21:24:55 +0100 Subject: [PATCH 231/725] Revert "Use upstream nlohmann_json" This reverts commit 4145cd2da002e1bd8affa0392c80118eabe58e3c. --- src/nlohmann/json.hpp | 20406 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 20406 insertions(+) create mode 100644 src/nlohmann/json.hpp diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp new file mode 100644 index 000000000..c9af0bed3 --- /dev/null +++ b/src/nlohmann/json.hpp @@ -0,0 +1,20406 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.5.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap + +// #include +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) + #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// #include + + +#include // not +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template +struct merge_and_renumber; + +template +struct merge_and_renumber, index_sequence> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template +using index_sequence_for = make_index_sequence; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // not +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include + +// #include + + +#include // random_access_iterator_tag + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include + +// #include + + +#include + +// #include + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template