Tagging release 2.27.1

-----BEGIN PGP SIGNATURE-----
 
 iQFHBAABCAAxFiEEtUHVUwEnDgvPFcpdgXC0cm1xmN4FAmfheacTHGVkb2xzdHJh
 QGdtYWlsLmNvbQAKCRCBcLRybXGY3kt2B/4tQvs6iDXA12d409ClHbVQjr1d0FLP
 rv8RxZ7Z4+Jaw8r2ra/I+gpr9juI5ULyEJWqfES72hTvbYPjH1Grsrrjak1tx57E
 +STs21oEPojE8LXsFH1oZamGPPIIpyQdxCvTgZs1N6cqUfCRQ3Jx97X6E6SIGJDR
 VqBM4ruSXCY57yT36HqwYydTkxzZHiNP5wwABGfSb7u9pYW5x3r8W7+fQ3udTnCw
 kCRhA5vnfxIQSlxu4j7dJqSCGzOIPnhYB19bXDV4aPhl4sn3pkBCdMZxPBlCWSwx
 it0ngMITf+TeiMpVl2TtvMBOHtlGrbhusbyKcsqzFYULGyGOC9ngTAY3
 =/JzB
 -----END PGP SIGNATURE-----

Merge tag '2.27.1' into detsys-main

Tagging release 2.27.1

src/build-remote/build-remote.cc

@@ -51,7 +51,7 @@ static bool allSupportedLocally(Store & store, const std::set<std::string>& requ
 static int main_build_remote(int argc, char * * argv)
 {
     {
-        logger = makeJSONLogger(*logger);
+        logger = makeJSONLogger(getStandardError());
 
         /* Ensure we don't get any SSH passphrase or host key popups. */
         unsetenv("DISPLAY");

src/libcmd/command.hh

@@ -347,7 +347,7 @@ struct MixEnvironment : virtual Args
     void setEnviron();
 };
 
-void completeFlakeInputPath(
+void completeFlakeInputAttrPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
     const std::vector<FlakeRef> & flakeRefs,

src/libcmd/common-eval-args.cc

@@ -33,8 +33,10 @@ EvalSettings evalSettings {
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
-                auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
-                return state.rootPath(state.store->toRealPath(storePath));
+                auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
+                auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+                state.allowPath(storePath);
+                return state.storePath(storePath);
             },
         },
     },
@@ -176,13 +178,15 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        return state.storePath(storePath);
     }
 
     else if (hasPrefix(s, "flake:")) {
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
+        auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+        state.allowPath(storePath);
+        return state.storePath(storePath);
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {

src/libcmd/installables.cc

@@ -33,7 +33,7 @@ namespace nix {
 
 namespace fs { using namespace std::filesystem; }
 
-void completeFlakeInputPath(
+void completeFlakeInputAttrPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
     const std::vector<FlakeRef> & flakeRefs,
@@ -117,10 +117,10 @@ MixFlakeOptions::MixFlakeOptions()
         .labels = {"input-path"},
         .handler = {[&](std::string s) {
             warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
-            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+            lockFlags.inputUpdates.insert(flake::parseInputAttrPath(s));
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+            completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
         }}
     });
 
@@ -129,15 +129,15 @@ MixFlakeOptions::MixFlakeOptions()
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
         .category = category,
         .labels = {"input-path", "flake-url"},
-        .handler = {[&](std::string inputPath, std::string flakeRef) {
+        .handler = {[&](std::string inputAttrPath, std::string flakeRef) {
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
-                flake::parseInputPath(inputPath),
+                flake::parseInputAttrPath(inputAttrPath),
                 parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
-                completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+                completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
             } else if (n == 1) {
                 completeFlakeRef(completions, getEvalState()->store, prefix);
             }

src/libcmd/misc-store-flags.cc

@@ -50,7 +50,7 @@ Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`).",
             .labels = {"hash-algo"},
             .handler = {[ha](std::string s) {
                 *ha = parseHashAlgo(s);
@@ -63,7 +63,7 @@ Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * o
 {
     return Args::Flag {
             .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
+            .description = "Hash algorithm (`blake3`, `md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
             .labels = {"hash-algo"},
             .handler = {[oha](std::string s) {
                 *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
@@ -120,7 +120,7 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method)
 
     - [`text`](@docroot@/store/store-object/content-address.md#method-text):
       Like `flat`, but used for
-      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
+      [derivations](@docroot@/glossary.md#gloss-store-derivation) serialized in store object and
       [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
       For advanced use-cases only;
       for regular usage prefer `nar` and `flat`.

src/libcmd/repl.cc

@@ -101,6 +101,8 @@ struct NixRepl
                               Value & v,
                               unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
     {
+        // Hide the progress bar during printing because it might interfere
+        auto suspension = logger->suspend();
         ::nix::printValue(*state, str, v, PrintOptions {
             .ansiColors = true,
             .force = true,
@@ -126,7 +128,7 @@ NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref<Store> store, ref<EvalS
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
-    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
+    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv))
     , runNixPtr{runNix}
     , interacter(make_unique<ReadlineLikeInteracter>(getDataDir() + "/repl-history"))
 {
@@ -138,16 +140,13 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
         out << ANSI_RED "error: " << ANSI_NORMAL;
     out << dt.hint.str() << "\n";
 
-    // prefer direct pos, but if noPos then try the expr.
-    auto pos = dt.pos
-        ? dt.pos
-        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
+    auto pos = dt.getPos(positions);
 
     if (pos) {
-        out << *pos;
-        if (auto loc = pos->getCodeLines()) {
+        out << pos;
+        if (auto loc = pos.getCodeLines()) {
             out << "\n";
-            printCodeLines(out, "", *pos, *loc);
+            printCodeLines(out, "", pos, *loc);
             out << "\n";
         }
     }
@@ -177,18 +176,20 @@ ReplExitStatus NixRepl::mainLoop()
 
     while (true) {
         // Hide the progress bar while waiting for user input, so that it won't interfere.
-        logger->pause();
-        // When continuing input from previous lines, don't print a prompt, just align to the same
-        // number of chars as the prompt.
-        if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
-            // Ctrl-D should exit the debugger.
-            state->debugStop = false;
-            logger->cout("");
-            // TODO: Should Ctrl-D exit just the current debugger session or
-            // the entire program?
-            return ReplExitStatus::QuitAll;
+        {
+            auto suspension = logger->suspend();
+            // When continuing input from previous lines, don't print a prompt, just align to the same
+            // number of chars as the prompt.
+            if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
+                // Ctrl-D should exit the debugger.
+                state->debugStop = false;
+                logger->cout("");
+                // TODO: Should Ctrl-D exit just the current debugger session or
+                // the entire program?
+                return ReplExitStatus::QuitAll;
+            }
+            // `suspension` resumes the logger
         }
-        logger->resume();
         try {
             switch (processLine(input)) {
                 case ProcessLineResult::Quit:
@@ -583,6 +584,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
+        auto suspension = logger->suspend();
         if (v.type() == nString) {
             std::cout << v.string_view();
         } else {
@@ -691,6 +693,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
         } else {
             Value v;
             evalString(line, v);
+            auto suspension = logger->suspend();
             printValue(std::cout, v, 1);
             std::cout << std::endl;
         }

src/libexpr-tests/error_traces.cc

@@ -1152,7 +1152,7 @@ namespace nix {
 
         ASSERT_TRACE1("hashString \"foo\" \"content\"",
                       UsageError,
-                      HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
+                      HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
 
         ASSERT_TRACE2("hashString \"sha256\" {}",
                       TypeError,

src/libexpr-tests/nix_api_expr.cc

@@ -172,7 +172,7 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build)
 
 TEST_F(nix_api_expr_test, nix_expr_realise_context)
 {
-    // TODO (ca-derivations): add a content-addressed derivation output, which produces a placeholder
+    // TODO (ca-derivations): add a content-addressing derivation output, which produces a placeholder
     auto expr = R"(
         ''
             a derivation output: ${

src/libexpr-tests/primops.cc

@@ -28,20 +28,15 @@ namespace nix {
     };
 
     class CaptureLogging {
-        Logger * oldLogger;
-        std::unique_ptr<CaptureLogger> tempLogger;
+        std::unique_ptr<Logger> oldLogger;
         public:
-            CaptureLogging() : tempLogger(std::make_unique<CaptureLogger>()) {
-                oldLogger = logger;
-                logger = tempLogger.get();
+            CaptureLogging() {
+                oldLogger = std::move(logger);
+                logger = std::make_unique<CaptureLogger>();
             }
 
             ~CaptureLogging() {
-                logger = oldLogger;
-            }
-
-            std::string get() const {
-                return tempLogger->get();
+                logger = std::move(oldLogger);
             }
     };
 
@@ -113,7 +108,7 @@ namespace nix {
         CaptureLogging l;
         auto v = eval("builtins.trace \"test string 123\" 123");
         ASSERT_THAT(v, IsIntEq(123));
-        auto text = l.get();
+        auto text = (dynamic_cast<CaptureLogger *>(logger.get()))->get();
         ASSERT_NE(text.find("test string 123"), std::string::npos);
     }
 

src/libexpr/call-flake.nix

@@ -23,7 +23,7 @@ let
   resolveInput =
     inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec;
 
-  # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+  # Follow an input attrpath (e.g. ["dwarffs" "nixpkgs"]) from the
   # root node, returning the final node.
   getInputByPath =
     nodeName: path:

src/libexpr/eval-error.cc

@@ -45,7 +45,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr
     // TODO: check compatibility with nested debugger calls.
     // TODO: What side-effects??
     error.state.debugTraces.push_front(DebugTrace{
-        .pos = error.state.positions[expr.getPos()],
+        .pos = expr.getPos(),
         .expr = expr,
         .env = env,
         .hint = HintFmt("Fake frame for debugging purposes"),

src/libexpr/eval-inline.hh

@@ -146,7 +146,7 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
 [[gnu::always_inline]]
 inline CallDepth EvalState::addCallDepth(const PosIdx pos) {
     if (callDepth > settings.maxCallDepth)
-        error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
+        error<EvalBaseError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
 
     return CallDepth(callDepth);
 };

src/libexpr/eval-settings.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include "config.hh"
-#include "ref.hh"
 #include "source-path.hh"
 
 namespace nix {

src/libexpr/eval.cc

@@ -246,15 +246,42 @@ EvalState::EvalState(
     , repair(NoRepair)
     , emptyBindings(0)
     , rootFS(
-        settings.restrictEval || settings.pureEval
-        ? ref<SourceAccessor>(AllowListSourceAccessor::create(getFSSourceAccessor(), {},
-            [&settings](const CanonPath & path) -> RestrictedPathError {
-                auto modeInformation = settings.pureEval
-                    ? "in pure evaluation mode (use '--impure' to override)"
-                    : "in restricted mode";
-                throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
-            }))
-        : getFSSourceAccessor())
+        ({
+            /* In pure eval mode, we provide a filesystem that only
+               contains the Nix store.
+
+               If we have a chroot store and pure eval is not enabled,
+               use a union accessor to make the chroot store available
+               at its logical location while still having the
+               underlying directory available. This is necessary for
+               instance if we're evaluating a file from the physical
+               /nix/store while using a chroot store. */
+            auto accessor = getFSSourceAccessor();
+
+            auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
+            if (settings.pureEval || store->storeDir != realStoreDir) {
+                auto storeFS = makeMountedSourceAccessor(
+                    {
+                        {CanonPath::root, makeEmptySourceAccessor()},
+                        {CanonPath(store->storeDir), makeFSSourceAccessor(realStoreDir)}
+                    });
+                accessor = settings.pureEval
+                    ? storeFS
+                    : makeUnionSourceAccessor({accessor, storeFS});
+            }
+
+            /* Apply access control if needed. */
+            if (settings.restrictEval || settings.pureEval)
+                accessor = AllowListSourceAccessor::create(accessor, {},
+                    [&settings](const CanonPath & path) -> RestrictedPathError {
+                        auto modeInformation = settings.pureEval
+                            ? "in pure evaluation mode (use '--impure' to override)"
+                            : "in restricted mode";
+                        throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
+                    });
+
+            accessor;
+        }))
     , corepkgsFS(make_ref<MemorySourceAccessor>())
     , internalFS(make_ref<MemorySourceAccessor>())
     , derivationInternal{corepkgsFS->addFile(
@@ -344,7 +371,7 @@ void EvalState::allowPath(const Path & path)
 void EvalState::allowPath(const StorePath & storePath)
 {
     if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
-        rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
+        rootFS2->allowPrefix(CanonPath(store->printStorePath(storePath)));
 }
 
 void EvalState::allowClosure(const StorePath & storePath)
@@ -422,16 +449,6 @@ void EvalState::checkURI(const std::string & uri)
 }
 
 
-Path EvalState::toRealPath(const Path & path, const NixStringContext & context)
-{
-    // FIXME: check whether 'path' is in 'context'.
-    return
-        !context.empty() && store->isInStore(path)
-        ? store->toRealPath(path)
-        : path;
-}
-
-
 Value * EvalState::addConstant(const std::string & name, Value & v, Constant info)
 {
     Value * v2 = allocValue();
@@ -754,18 +771,26 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     if (!debugRepl || inDebugger)
         return;
 
-    auto dts =
-        error && expr.getPos()
-        ? std::make_unique<DebugTraceStacker>(
-            *this,
-            DebugTrace {
-                .pos = error->info().pos ? error->info().pos : positions[expr.getPos()],
+    auto dts = [&]() -> std::unique_ptr<DebugTraceStacker> {
+        if (error && expr.getPos()) {
+            auto trace = DebugTrace{
+                .pos = [&]() -> std::variant<Pos, PosIdx> {
+                    if (error->info().pos) {
+                        if (auto * pos = error->info().pos.get())
+                            return *pos;
+                        return noPos;
+                    }
+                    return expr.getPos();
+                }(),
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
-                .isError = true
-            })
-        : nullptr;
+                .isError = true};
+
+            return std::make_unique<DebugTraceStacker>(*this, std::move(trace));
+        }
+        return nullptr;
+    }();
 
     if (error)
     {
@@ -810,7 +835,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::shared_ptr<Pos> && pos,
+    std::variant<Pos, PosIdx> pos,
     const Args & ... formatArgs)
 {
     return std::make_unique<DebugTraceStacker>(state,
@@ -1087,7 +1112,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
+                e->getPos(),
                 "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
@@ -1313,9 +1338,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
             state,
             *this,
             env2,
-            getPos()
-                ? std::make_shared<Pos>(state.positions[getPos()])
-                : nullptr,
+            getPos(),
             "while evaluating a '%1%' expression",
             "let"
         )
@@ -1384,7 +1407,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 state,
                 *this,
                 env,
-                state.positions[getPos()],
+                getPos(),
                 "while evaluating the attribute '%1%'",
                 showAttrPath(state, env, attrPath))
             : nullptr;
@@ -1585,7 +1608,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
             try {
                 auto dts = debugRepl
                     ? makeDebugTraceStacker(
-                        *this, *lambda.body, env2, positions[lambda.pos],
+                        *this, *lambda.body, env2, lambda.pos,
                         "while calling %s",
                         lambda.name
                         ? concatStrings("'", symbols[lambda.name], "'")
@@ -1720,9 +1743,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
             state,
             *this,
             env,
-            getPos()
-                ? std::make_shared<Pos>(state.positions[getPos()])
-                : nullptr,
+            getPos(),
             "while calling a function"
         )
         : nullptr;
@@ -2051,7 +2072,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
-        v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
+        v.mkPath(state.rootPath(CanonPath(str())));
     } else
         v.mkStringMove(c_str(), context);
 }
@@ -2106,7 +2127,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     // If the value is a thunk, we're evaling. Otherwise no trace necessary.
                     auto dts = debugRepl && i.value->isThunk()
-                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos],
+                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, i.pos,
                             "while evaluating the attribute '%1%'", symbols[i.name])
                         : nullptr;
 
@@ -2432,7 +2453,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
     auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
     if (path == "" || path[0] != '/')
         error<EvalError>("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow();
-    return rootPath(CanonPath(path));
+    return rootPath(path);
 }
 
 
@@ -3086,7 +3107,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
-            return finish(rootPath(store->toRealPath(storePath)));
+            return finish(this->storePath(storePath));
         } catch (Error & e) {
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)

src/libexpr/eval.hh

@@ -171,11 +171,28 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<Pos> pos;
+    /* WARNING: Converting PosIdx -> Pos should be done with extra care. This is
+       due to the fact that operator[] of PosTable is incredibly expensive. */
+    std::variant<Pos, PosIdx> pos;
     const Expr & expr;
     const Env & env;
     HintFmt hint;
     bool isError;
+
+    Pos getPos(const PosTable & table) const
+    {
+        return std::visit(
+            overloaded{
+                [&](PosIdx idx) {
+                    // Prefer direct pos, but if noPos then try the expr.
+                    if (!idx)
+                        idx = expr.getPos();
+                    return table[idx];
+                },
+                [&](Pos pos) { return pos; },
+            },
+            pos);
+    }
 };
 
 class EvalState : public std::enable_shared_from_this<EvalState>
@@ -389,6 +406,15 @@ public:
      */
     SourcePath rootPath(PathView path);
 
+    /**
+     * Return a `SourcePath` that refers to `path` in the store.
+     *
+     * For now, this has to also be within the root filesystem for
+     * backwards compat, but for Windows and maybe also pure eval, we'll
+     * probably want to do something different.
+     */
+    SourcePath storePath(const StorePath & path);
+
     /**
      * Allow access to a path.
      */
@@ -412,17 +438,6 @@ public:
 
     void checkURI(const std::string & uri);
 
-    /**
-     * When using a diverted store and 'path' is in the Nix store, map
-     * 'path' to the diverted location (e.g. /nix/store/foo is mapped
-     * to /home/alice/my-nix/nix/store/foo). However, this is only
-     * done if the context is not empty, since otherwise we're
-     * probably trying to read from the actual /nix/store. This is
-     * intended to distinguish between import-from-derivation and
-     * sources stored in the actual /nix/store.
-     */
-    Path toRealPath(const Path & path, const NixStringContext & context);
-
     /**
      * Parse a Nix expression from the specified file.
      */

src/libexpr/meson.build

@@ -24,6 +24,7 @@ deps_public_maybe_subproject = [
   dependency('nix-fetchers'),
 ]
 subdir('nix-meson-build-support/subprojects')
+subdir('nix-meson-build-support/big-objs')
 
 boost = dependency(
   'boost',
@@ -171,8 +172,6 @@ headers = [config_h] + files(
   # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',
-  'pos-idx.hh',
-  'pos-table.hh',
   'primops.hh',
   'print-ambiguous.hh',
   'print-options.hh',

src/libexpr/nixexpr.cc

@@ -310,7 +310,7 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     const StaticEnv * curEnv;
     Level level;
     int withLevel = -1;
-    for (curEnv = env.get(), level = 0; curEnv; curEnv = curEnv->up, level++) {
+    for (curEnv = env.get(), level = 0; curEnv; curEnv = curEnv->up.get(), level++) {
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
@@ -331,7 +331,7 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
             "undefined variable '%1%'",
             es.symbols[name]
         ).atPos(pos).debugThrow();
-    for (auto * e = env.get(); e && !fromWith; e = e->up)
+    for (auto * e = env.get(); e && !fromWith; e = e->up.get())
         fromWith = e->isWith;
     this->level = withLevel;
 }
@@ -379,7 +379,7 @@ std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
     // and displacement, and nothing else is allowed to access it. ideally we'd
     // not even *have* an expr that grabs anything from this env since it's fully
     // invisible, but the evaluator does not allow for this yet.
-    auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
+    auto inner = std::make_shared<StaticEnv>(nullptr, env, 0);
     for (auto from : *inheritFromExprs)
         from->bindVars(es, env);
 
@@ -393,7 +393,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
 
     if (recursive) {
         auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
-            auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
 
             Displacement displ = 0;
             for (auto & i : attrs)
@@ -440,7 +440,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     auto newEnv = std::make_shared<StaticEnv>(
-        nullptr, env.get(),
+        nullptr, env,
         (hasFormals() ? formals->formals.size() : 0) +
         (!arg ? 0 : 1));
 
@@ -474,7 +474,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
-        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
 
         Displacement displ = 0;
         for (auto & i : attrs->attrs)
@@ -500,7 +500,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     parentWith = nullptr;
-    for (auto * e = env.get(); e && !parentWith; e = e->up)
+    for (auto * e = env.get(); e && !parentWith; e = e->up.get())
         parentWith = e->isWith;
 
     /* Does this `with' have an enclosing `with'?  If so, record its
@@ -509,14 +509,14 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     const StaticEnv * curEnv;
     Level level;
     prevWith = 0;
-    for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up, level++)
+    for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up.get(), level++)
         if (curEnv->isWith) {
             prevWith = level;
             break;
         }
 
     attrs->bindVars(es, env);
-    auto newEnv = std::make_shared<StaticEnv>(this, env.get());
+    auto newEnv = std::make_shared<StaticEnv>(this, env);
     body->bindVars(es, newEnv);
 }
 
@@ -601,41 +601,6 @@ void ExprLambda::setDocComment(DocComment docComment) {
     }
 };
 
-
-
-/* Position table. */
-
-Pos PosTable::operator[](PosIdx p) const
-{
-    auto origin = resolve(p);
-    if (!origin)
-        return {};
-
-    const auto offset = origin->offsetOf(p);
-
-    Pos result{0, 0, origin->origin};
-    auto lines = this->lines.lock();
-    auto linesForInput = (*lines)[origin->offset];
-
-    if (linesForInput.empty()) {
-        auto source = result.getSource().value_or("");
-        const char * begin = source.data();
-        for (Pos::LinesIterator it(source), end; it != end; it++)
-            linesForInput.push_back(it->data() - begin);
-        if (linesForInput.empty())
-            linesForInput.push_back(0);
-    }
-    // as above: the first line starts at byte 0 and is always present
-    auto lineStartOffset = std::prev(
-        std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
-
-    result.line = 1 + (lineStartOffset - linesForInput.begin());
-    result.column = 1 + (offset - *lineStartOffset);
-    return result;
-}
-
-
-
 /* Symbol table. */
 
 size_t SymbolTable::totalSize() const

src/libexpr/nixexpr.hh

@@ -480,13 +480,16 @@ extern ExprBlackHole eBlackHole;
 struct StaticEnv
 {
     ExprWith * isWith;
-    const StaticEnv * up;
+    std::shared_ptr<const StaticEnv> up;
 
     // Note: these must be in sorted order.
     typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
 
-    StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+    StaticEnv(ExprWith * isWith, std::shared_ptr<const StaticEnv> up, size_t expectedSize = 0)
+        : isWith(isWith)
+        , up(std::move(up))
+    {
         vars.reserve(expectedSize);
     };
 

src/libexpr/parser.y

@@ -359,11 +359,18 @@ string_parts_interpolated
 
 path_start
   : PATH {
-    Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs()));
+    std::string_view literal({$1.p, $1.l});
+    Path path(absPath(literal, state->basePath.path.abs()));
     /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+    if (literal.size() > 1 && literal.back() == '/')
+      path += '/';
+    $$ =
+        /* Absolute paths are always interpreted relative to the
+           root filesystem accessor, rather than the accessor of the
+           current Nix expression. */
+        literal.front() == '/'
+        ? new ExprPath(state->rootFS, std::move(path))
+        : new ExprPath(state->basePath.accessor, std::move(path));
   }
   | HPATH {
     if (state->settings.pureEval) {

src/libexpr/paths.cc

@@ -1,3 +1,4 @@
+#include "store-api.hh"
 #include "eval.hh"
 
 namespace nix {
@@ -12,4 +13,9 @@ SourcePath EvalState::rootPath(PathView path)
     return {rootFS, CanonPath(absPath(path))};
 }
 
+SourcePath EvalState::storePath(const StorePath & path)
+{
+    return {rootFS, CanonPath{store->printStorePath(path)}};
+}
+
 }

src/libexpr/primops.cc

@@ -145,8 +145,7 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, st
     try {
         if (!context.empty() && path.accessor == state.rootFS) {
             auto rewrites = state.realiseContext(context);
-            auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
-            path = {path.accessor, CanonPath(realPath)};
+            path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
         }
         return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
     } catch (Error & e) {
@@ -239,7 +238,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
     Env * env = &state.allocEnv(vScope->attrs()->size());
     env->up = &state.baseEnv;
 
-    auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv.get(), vScope->attrs()->size());
+    auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv, vScope->attrs()->size());
 
     unsigned int displ = 0;
     for (auto & attr : *vScope->attrs()) {
@@ -1595,9 +1594,13 @@ static RegisterPrimOp primop_placeholder({
     .name = "placeholder",
     .args = {"output"},
     .doc = R"(
-      Return a placeholder string for the specified *output* that will be
-      substituted by the corresponding output path at build time. Typical
-      outputs would be `"out"`, `"bin"` or `"dev"`.
+      Return at
+      [output placeholder string](@docroot@/store/derivation/index.md#output-placeholder)
+      for the specified *output* that will be substituted by the corresponding
+      [output path](@docroot@/glossary.md#gloss-output-path)
+      at build time.
+
+      Typical outputs would be `"out"`, `"bin"` or `"dev"`.
     )",
     .fun = prim_placeholder,
 });
@@ -2135,12 +2138,15 @@ static RegisterPrimOp primop_outputOf({
     .name = "__outputOf",
     .args = {"derivation-reference", "output-name"},
     .doc = R"(
-      Return the output path of a derivation, literally or using a placeholder if needed.
+      Return the output path of a derivation, literally or using an
+      [input placeholder string](@docroot@/store/derivation/index.md#input-placeholder)
+      if needed.
 
       If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned.
-      But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
+      But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), an input placeholder will be returned instead.
 
-      *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
+      *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be an input placeholder reference.
+      If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
       This primop can be chained arbitrarily deeply.
       For instance,
 
@@ -2150,9 +2156,9 @@ static RegisterPrimOp primop_outputOf({
         "out"
       ```
 
-      will return a placeholder for the output of the output of `myDrv`.
+      will return a input placeholder for the output of the output of `myDrv`.
 
-      This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
+      This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-paths), e.g. as part of installable syntax on the command line.
     )",
     .fun = prim_outputOf,
     .experimentalFeature = Xp::DynamicDerivations,
@@ -2472,21 +2478,11 @@ static void addPath(
     const NixStringContext & context)
 {
     try {
-        StorePathSet refs;
-
         if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) {
             // FIXME: handle CA derivation outputs (where path needs to
             // be rewritten to the actual output).
             auto rewrites = state.realiseContext(context);
-            path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))};
-
-            try {
-                auto [storePath, subPath] = state.store->toStorePath(path.path.abs());
-                // FIXME: we should scanForReferences on the path before adding it
-                refs = state.store->queryPathInfo(storePath)->references;
-                path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)};
-            } catch (Error &) { // FIXME: should be InvalidPathError
-            }
+            path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
         }
 
         std::unique_ptr<PathFilter> filter;

src/libexpr/primops/fetchTree.cc

@@ -90,24 +90,26 @@ static void fetchTree(
     fetchers::Input input { state.fetchSettings };
     NixStringContext context;
     std::optional<std::string> type;
+    auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree";
     if (params.isFetchGit) type = "git";
 
     state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
-        state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchTree");
+        state.forceAttrs(*args[0], pos, fmt("while evaluating the argument passed to '%s'", fetcher));
 
         fetchers::Attrs attrs;
 
         if (auto aType = args[0]->attrs()->get(state.sType)) {
             if (type)
                 state.error<EvalError>(
-                    "unexpected attribute 'type'"
+                    "unexpected argument 'type'"
                 ).atPos(pos).debugThrow();
-            type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree");
+            type = state.forceStringNoCtx(*aType->value, aType->pos,
+                fmt("while evaluating the `type` argument passed to '%s'", fetcher));
         } else if (!type)
             state.error<EvalError>(
-                "attribute 'type' is missing in call to 'fetchTree'"
+                "argument 'type' is missing in call to '%s'", fetcher
             ).atPos(pos).debugThrow();
 
         attrs.emplace("type", type.value());
@@ -127,9 +129,8 @@ static void fetchTree(
             else if (attr.value->type() == nInt) {
                 auto intValue = attr.value->integer().value;
 
-                if (intValue < 0) {
-                    state.error<EvalError>("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow();
-                }
+                if (intValue < 0)
+                    state.error<EvalError>("negative value given for '%s' argument '%s': %d", fetcher, state.symbols[attr.name], intValue).atPos(pos).debugThrow();
 
                 attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
             } else if (state.symbols[attr.name] == "publicKeys") {
@@ -137,8 +138,8 @@ static void fetchTree(
                 attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump());
             }
             else
-                state.error<TypeError>("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
-                    state.symbols[attr.name], showType(*attr.value)).debugThrow();
+                state.error<TypeError>("argument '%s' to '%s' is %s while a string, Boolean or integer is expected",
+                    state.symbols[attr.name], fetcher, showType(*attr.value)).debugThrow();
         }
 
         if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
@@ -153,14 +154,14 @@ static void fetchTree(
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>(
-                    "attribute 'name' isn’t supported in call to 'fetchTree'"
+                    "argument 'name' isn’t supported in call to '%s'", fetcher
                 ).atPos(pos).debugThrow();
 
         input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context,
-                "while evaluating the first argument passed to the fetcher",
-                false, false).toOwned();
+            fmt("while evaluating the first argument passed to '%s'", fetcher),
+            false, false).toOwned();
 
         if (params.isFetchGit) {
             fetchers::Attrs attrs;
@@ -178,15 +179,16 @@ static void fetchTree(
     if (!state.settings.pureEval && !input.isDirect())
         input = lookupInRegistries(state.store, input).first;
 
-    if (state.settings.pureEval && !input.isConsideredLocked(state.fetchSettings)) {
-        auto fetcher = "fetchTree";
-        if (params.isFetchGit)
-            fetcher = "fetchGit";
-
-        state.error<EvalError>(
-            "in pure evaluation mode, '%s' will not fetch unlocked input '%s'",
-            fetcher, input.to_string()
-        ).atPos(pos).debugThrow();
+    if (state.settings.pureEval && !input.isLocked()) {
+        if (input.getNarHash())
+            warn(
+                "Input '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
+                "This is deprecated since such inputs are verifiable but may not be reproducible.",
+                input.to_string());
+        else
+            state.error<EvalError>(
+                "in pure evaluation mode, '%s' will not fetch unlocked input '%s'",
+                fetcher, input.to_string()).atPos(pos).debugThrow();
     }
 
     state.checkURI(input.toURLString());
@@ -361,6 +363,12 @@ static RegisterPrimOp primop_fetchTree({
 
           Default: `false`
 
+        - `lfs` (Bool, optional)
+
+          Fetch any [Git LFS](https://git-lfs.com/) files.
+
+          Default: `false`
+
         - `allRefs` (Bool, optional)
 
           By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
@@ -683,6 +691,13 @@ static RegisterPrimOp primop_fetchGit({
 
         Make a shallow clone when fetching the Git tree.
         When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+      - `lfs` (default: `false`)
+
+        A boolean that when `true` specifies that [Git LFS] files should be fetched.
+
+        [Git LFS]: https://git-lfs.com/
+
       - `allRefs`
 
         Whether to fetch all references (eg. branches and tags) of the repository.

src/libfetchers-tests/access-tokens.cc

@@ -0,0 +1,97 @@
+#include <gtest/gtest.h>
+#include "fetchers.hh"
+#include "fetch-settings.hh"
+#include "json-utils.hh"
+#include <nlohmann/json.hpp>
+#include "tests/characterization.hh"
+
+namespace nix::fetchers {
+
+using nlohmann::json;
+
+class AccessKeysTest : public ::testing::Test
+{
+protected:
+
+public:
+    void SetUp() override {}
+    void TearDown() override {}
+};
+
+TEST_F(AccessKeysTest, singleOrgGitHub)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com/a", "token"});
+    auto i = Input::fromURL(fetchSettings, "github:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b");
+    ASSERT_EQ(token, "token");
+}
+
+TEST_F(AccessKeysTest, nonMatches)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com", "token"});
+    auto i = Input::fromURL(fetchSettings, "gitlab:github.com/evil");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/github.com/evil");
+    ASSERT_EQ(token, std::nullopt);
+}
+
+TEST_F(AccessKeysTest, noPartialMatches)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com/partial", "token"});
+    auto i = Input::fromURL(fetchSettings, "github:partial-match/repo");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/partial-match");
+    ASSERT_EQ(token, std::nullopt);
+}
+
+TEST_F(AccessKeysTest, repoGitHub)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com", "token"});
+    fetchSettings.accessTokens.get().insert({"github.com/a/b", "another_token"});
+    fetchSettings.accessTokens.get().insert({"github.com/a/c", "yet_another_token"});
+    auto i = Input::fromURL(fetchSettings, "github:a/a");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/a");
+    ASSERT_EQ(token, "token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b");
+    ASSERT_EQ(token, "another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/c");
+    ASSERT_EQ(token, "yet_another_token");
+}
+
+TEST_F(AccessKeysTest, multipleGitLab)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"gitlab.com", "token"});
+    fetchSettings.accessTokens.get().insert({"gitlab.com/a/b", "another_token"});
+    auto i = Input::fromURL(fetchSettings, "gitlab:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/b");
+    ASSERT_EQ(token, "another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/c");
+    ASSERT_EQ(token, "token");
+}
+
+TEST_F(AccessKeysTest, multipleSourceHut)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"git.sr.ht", "token"});
+    fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b", "another_token"});
+    auto i = Input::fromURL(fetchSettings, "sourcehut:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/b");
+    ASSERT_EQ(token, "another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/c");
+    ASSERT_EQ(token, "token");
+}
+
+}

src/libfetchers-tests/git-utils.cc

@@ -7,13 +7,18 @@
 #include <gtest/gtest.h>
 #include "fs-sink.hh"
 #include "serialise.hh"
+#include "git-lfs-fetch.hh"
 
 namespace nix {
 
+namespace fs {
+using namespace std::filesystem;
+}
+
 class GitUtilsTest : public ::testing::Test
 {
     // We use a single repository for all tests.
-    Path tmpDir;
+    fs::path tmpDir;
     std::unique_ptr<AutoDelete> delTmpDir;
 
 public:
@@ -25,7 +30,7 @@ public:
         // Create the repo with libgit2
         git_libgit2_init();
         git_repository * repo = nullptr;
-        auto r = git_repository_init(&repo, tmpDir.c_str(), 0);
+        auto r = git_repository_init(&repo, tmpDir.string().c_str(), 0);
         ASSERT_EQ(r, 0);
         git_repository_free(repo);
     }
@@ -41,6 +46,11 @@ public:
     {
         return GitRepo::openRepo(tmpDir, true, false);
     }
+
+    std::string getRepoName() const
+    {
+        return tmpDir.filename().string();
+    }
 };
 
 void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable)
@@ -78,7 +88,7 @@ TEST_F(GitUtilsTest, sink_basic)
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
-    auto accessor = repo->getAccessor(result, false);
+    auto accessor = repo->getAccessor(result, false, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

src/libfetchers-tests/meson.build

@@ -31,6 +31,9 @@ deps_private += rapidcheck
 gtest = dependency('gtest', main : true)
 deps_private += gtest
 
+libgit2 = dependency('libgit2')
+deps_private += libgit2
+
 add_project_arguments(
   # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
   # It would be nice for our headers to be idempotent instead.
@@ -43,6 +46,8 @@ add_project_arguments(
 subdir('nix-meson-build-support/common')
 
 sources = files(
+  'access-tokens.cc',
+  'git-utils.cc',
   'public-key.cc',
 )
 

src/libfetchers-tests/package.nix

@@ -7,6 +7,7 @@
   nix-fetchers,
   nix-store-test-support,
 
+  libgit2,
   rapidcheck,
   gtest,
   runCommand,
@@ -42,6 +43,7 @@ mkMesonExecutable (finalAttrs: {
     nix-store-test-support
     rapidcheck
     gtest
+    libgit2
   ];
 
   mesonFlags = [

src/libfetchers/fetch-settings.hh

@@ -23,9 +23,11 @@ struct Settings : public Config
           Access tokens are specified as a string made up of
           space-separated `host=token` values.  The specific token
           used is selected by matching the `host` portion against the
-          "host" specification of the input. The actual use of the
-          `token` value is determined by the type of resource being
-          accessed:
+          "host" specification of the input. The `host` portion may
+          contain a path element which will match against the prefix
+          URL for the input. (eg: `github.com/org=token`). The actual use
+          of the `token` value is determined by the type of resource
+          being accessed:
 
           * Github: the token value is the OAUTH-TOKEN string obtained
             as the Personal Access Token from the Github server (see

src/libfetchers/fetchers.cc

@@ -155,12 +155,6 @@ bool Input::isLocked() const
     return scheme && scheme->isLocked(*this);
 }
 
-bool Input::isConsideredLocked(
-    const Settings & settings) const
-{
-    return isLocked() || (settings.allowDirtyLocks && getNarHash());
-}
-
 bool Input::isFinal() const
 {
     return maybeGetBoolAttr(attrs, "__final").value_or(false);
@@ -192,6 +186,7 @@ bool Input::contains(const Input & other) const
     return false;
 }
 
+// FIXME: remove
 std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
 {
     if (!scheme)
@@ -206,10 +201,6 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
             auto narHash = store->queryPathInfo(storePath)->narHash;
             result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-            // FIXME: we would like to mark inputs as final in
-            // getAccessorUnchecked(), but then we can't add
-            // narHash. Or maybe narHash should be excluded from the
-            // concept of "final" inputs?
             result.attrs.insert_or_assign("__final", Explicit<bool>(true));
 
             assert(result.isFinal());
@@ -290,6 +281,8 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
     try {
         auto [accessor, result] = getAccessorUnchecked(store);
 
+        result.attrs.insert_or_assign("__final", Explicit<bool>(true));
+
         checkLocks(*this, result);
 
         return {accessor, std::move(result)};

src/libfetchers/fetchers.hh

@@ -90,15 +90,6 @@ public:
      */
     bool isLocked() const;
 
-    /**
-     * Return whether the input is either locked, or, if
-     * `allow-dirty-locks` is enabled, it has a NAR hash. In the
-     * latter case, we can verify the input but we may not be able to
-     * fetch it from anywhere.
-     */
-    bool isConsideredLocked(
-        const Settings & settings) const;
-
     /**
      * Only for relative path flakes, i.e. 'path:./foo', returns the
      * relative path, i.e. './foo'.
@@ -273,6 +264,9 @@ struct InputScheme
 
     virtual std::optional<std::string> isRelative(const Input & input) const
     { return std::nullopt; }
+
+    virtual std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const
+    { return {};}
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

src/libfetchers/git-lfs-fetch.cc

@@ -0,0 +1,279 @@
+#include "git-lfs-fetch.hh"
+#include "git-utils.hh"
+#include "filetransfer.hh"
+#include "processes.hh"
+#include "url.hh"
+#include "users.hh"
+#include "hash.hh"
+
+#include <git2/attr.h>
+#include <git2/config.h>
+#include <git2/errors.h>
+#include <git2/remote.h>
+
+#include <nlohmann/json.hpp>
+
+namespace nix::lfs {
+
+// if authHeader is "", downloadToSink assumes no auth is expected
+static void downloadToSink(
+    const std::string & url,
+    const std::string & authHeader,
+    // FIXME: passing a StringSink is superfluous, we may as well
+    // return a string. Or use an abstract Sink for streaming.
+    StringSink & sink,
+    std::string sha256Expected,
+    size_t sizeExpected)
+{
+    FileTransferRequest request(url);
+    Headers headers;
+    if (!authHeader.empty())
+        headers.push_back({"Authorization", authHeader});
+    request.headers = headers;
+    getFileTransfer()->download(std::move(request), sink);
+
+    auto sizeActual = sink.s.length();
+    if (sizeExpected != sizeActual)
+        throw Error("size mismatch while fetching %s: expected %d but got %d", url, sizeExpected, sizeActual);
+
+    auto sha256Actual = hashString(HashAlgorithm::SHA256, sink.s).to_string(HashFormat::Base16, false);
+    if (sha256Actual != sha256Expected)
+        throw Error(
+            "hash mismatch while fetching %s: expected sha256:%s but got sha256:%s", url, sha256Expected, sha256Actual);
+}
+
+static std::string getLfsApiToken(const ParsedURL & url)
+{
+    auto [status, output] = runProgram(RunOptions{
+        .program = "ssh",
+        .args = {*url.authority, "git-lfs-authenticate", url.path, "download"},
+    });
+
+    if (output.empty())
+        throw Error(
+            "git-lfs-authenticate: no output (cmd: ssh %s git-lfs-authenticate %s download)",
+            url.authority.value_or(""),
+            url.path);
+
+    auto queryResp = nlohmann::json::parse(output);
+    if (!queryResp.contains("header"))
+        throw Error("no header in git-lfs-authenticate response");
+    if (!queryResp["header"].contains("Authorization"))
+        throw Error("no Authorization in git-lfs-authenticate response");
+
+    return queryResp["header"]["Authorization"].get<std::string>();
+}
+
+typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
+typedef std::unique_ptr<git_config_entry, Deleter<git_config_entry_free>> GitConfigEntry;
+
+static std::string getLfsEndpointUrl(git_repository * repo)
+{
+    GitConfig config;
+    if (git_repository_config(Setter(config), repo)) {
+        GitConfigEntry entry;
+        if (!git_config_get_entry(Setter(entry), config.get(), "lfs.url")) {
+            auto value = std::string(entry->value);
+            if (!value.empty()) {
+                debug("Found explicit lfs.url value: %s", value);
+                return value;
+            }
+        }
+    }
+
+    git_remote * remote = nullptr;
+    if (git_remote_lookup(&remote, repo, "origin"))
+        return "";
+
+    const char * url_c_str = git_remote_url(remote);
+    if (!url_c_str)
+        return "";
+
+    return std::string(url_c_str);
+}
+
+static std::optional<Pointer> parseLfsPointer(std::string_view content, std::string_view filename)
+{
+    // https://github.com/git-lfs/git-lfs/blob/2ef4108/docs/spec.md
+    //
+    // example git-lfs pointer file:
+    // version https://git-lfs.github.com/spec/v1
+    // oid sha256:f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf
+    // size 10000000
+    // (ending \n)
+
+    if (!content.starts_with("version ")) {
+        // Invalid pointer file
+        return std::nullopt;
+    }
+
+    if (!content.starts_with("version https://git-lfs.github.com/spec/v1")) {
+        // In case there's new spec versions in the future, but for now only v1 exists
+        debug("Invalid version found on potential lfs pointer file, skipping");
+        return std::nullopt;
+    }
+
+    std::string oid;
+    std::string size;
+
+    for (auto & line : tokenizeString<Strings>(content, "\n")) {
+        if (line.starts_with("version ")) {
+            continue;
+        }
+        if (line.starts_with("oid sha256:")) {
+            oid = line.substr(11); // skip "oid sha256:"
+            continue;
+        }
+        if (line.starts_with("size ")) {
+            size = line.substr(5); // skip "size "
+            continue;
+        }
+
+        debug("Custom extension '%s' found, ignoring", line);
+    }
+
+    if (oid.length() != 64 || !std::all_of(oid.begin(), oid.end(), ::isxdigit)) {
+        debug("Invalid sha256 %s, skipping", oid);
+        return std::nullopt;
+    }
+
+    if (size.length() == 0 || !std::all_of(size.begin(), size.end(), ::isdigit)) {
+        debug("Invalid size %s, skipping", size);
+        return std::nullopt;
+    }
+
+    return std::make_optional(Pointer{oid, std::stoul(size)});
+}
+
+Fetch::Fetch(git_repository * repo, git_oid rev)
+{
+    this->repo = repo;
+    this->rev = rev;
+
+    const auto remoteUrl = lfs::getLfsEndpointUrl(repo);
+
+    this->url = nix::parseURL(nix::fixGitURL(remoteUrl)).canonicalise();
+}
+
+bool Fetch::shouldFetch(const CanonPath & path) const
+{
+    const char * attr = nullptr;
+    git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
+    opts.attr_commit_id = this->rev;
+    opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM;
+    if (git_attr_get_ext(&attr, (git_repository *) (this->repo), &opts, path.rel_c_str(), "filter"))
+        throw Error("cannot get git-lfs attribute: %s", git_error_last()->message);
+    debug("Git filter for '%s' is '%s'", path, attr ? attr : "null");
+    return attr != nullptr && !std::string(attr).compare("lfs");
+}
+
+static nlohmann::json pointerToPayload(const std::vector<Pointer> & items)
+{
+    nlohmann::json jArray = nlohmann::json::array();
+    for (const auto & pointer : items)
+        jArray.push_back({{"oid", pointer.oid}, {"size", pointer.size}});
+    return jArray;
+}
+
+std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointers) const
+{
+    ParsedURL httpUrl(url);
+    httpUrl.scheme = url.scheme == "ssh" ? "https" : url.scheme;
+    FileTransferRequest request(httpUrl.to_string() + "/info/lfs/objects/batch");
+    request.post = true;
+    Headers headers;
+    if (this->url.scheme == "ssh")
+        headers.push_back({"Authorization", lfs::getLfsApiToken(this->url)});
+    headers.push_back({"Content-Type", "application/vnd.git-lfs+json"});
+    headers.push_back({"Accept", "application/vnd.git-lfs+json"});
+    request.headers = headers;
+    nlohmann::json oidList = pointerToPayload(pointers);
+    nlohmann::json data = {{"operation", "download"}};
+    data["objects"] = oidList;
+    request.data = data.dump();
+
+    FileTransferResult result = getFileTransfer()->upload(request);
+    auto responseString = result.data;
+
+    std::vector<nlohmann::json> objects;
+    // example resp here:
+    // {"objects":[{"oid":"f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf","size":10000000,"actions":{"download":{"href":"https://gitlab.com/b-camacho/test-lfs.git/gitlab-lfs/objects/f5e02aa71e67f41d79023a128ca35bad86cf7b6656967bfe0884b3a3c4325eaf","header":{"Authorization":"Basic
+    // Yi1jYW1hY2hvOmV5SjBlWEFpT2lKS1YxUWlMQ0poYkdjaU9pSklVekkxTmlKOS5leUprWVhSaElqcDdJbUZqZEc5eUlqb2lZaTFqWVcxaFkyaHZJbjBzSW1wMGFTSTZJbUptTURZNFpXVTFMVEprWmpVdE5HWm1ZUzFpWWpRMExUSXpNVEV3WVRReU1qWmtaaUlzSW1saGRDSTZNVGN4TkRZeE16ZzBOU3dpYm1KbUlqb3hOekUwTmpFek9EUXdMQ0psZUhBaU9qRTNNVFEyTWpFd05EVjkuZk9yMDNkYjBWSTFXQzFZaTBKRmJUNnJTTHJPZlBwVW9lYllkT0NQZlJ4QQ=="}}},"authenticated":true}]}
+
+    try {
+        auto resp = nlohmann::json::parse(responseString);
+        if (resp.contains("objects"))
+            objects.insert(objects.end(), resp["objects"].begin(), resp["objects"].end());
+        else
+            throw Error("response does not contain 'objects'");
+
+        return objects;
+    } catch (const nlohmann::json::parse_error & e) {
+        printMsg(lvlTalkative, "Full response: '%1%'", responseString);
+        throw Error("response did not parse as json: %s", e.what());
+    }
+}
+
+void Fetch::fetch(
+    const std::string & content,
+    const CanonPath & pointerFilePath,
+    StringSink & sink,
+    std::function<void(uint64_t)> sizeCallback) const
+{
+    debug("trying to fetch '%s' using git-lfs", pointerFilePath);
+
+    if (content.length() >= 1024) {
+        warn("encountered file '%s' that should have been a git-lfs pointer, but is too large", pointerFilePath);
+        sizeCallback(content.length());
+        sink(content);
+        return;
+    }
+
+    const auto pointer = parseLfsPointer(content, pointerFilePath.rel());
+    if (pointer == std::nullopt) {
+        warn("encountered file '%s' that should have been a git-lfs pointer, but is invalid", pointerFilePath);
+        sizeCallback(content.length());
+        sink(content);
+        return;
+    }
+
+    Path cacheDir = getCacheDir() + "/git-lfs";
+    std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false)
+                      + "/" + pointer->oid;
+    Path cachePath = cacheDir + "/" + key;
+    if (pathExists(cachePath)) {
+        debug("using cache entry %s -> %s", key, cachePath);
+        sink(readFile(cachePath));
+        return;
+    }
+    debug("did not find cache entry for %s", key);
+
+    std::vector<Pointer> pointers;
+    pointers.push_back(pointer.value());
+    const auto objUrls = fetchUrls(pointers);
+
+    const auto obj = objUrls[0];
+    try {
+        std::string sha256 = obj.at("oid"); // oid is also the sha256
+        std::string ourl = obj.at("actions").at("download").at("href");
+        std::string authHeader = "";
+        if (obj.at("actions").at("download").contains("header")
+            && obj.at("actions").at("download").at("header").contains("Authorization")) {
+            authHeader = obj["actions"]["download"]["header"]["Authorization"];
+        }
+        const uint64_t size = obj.at("size");
+        sizeCallback(size);
+        downloadToSink(ourl, authHeader, sink, sha256, size);
+
+        debug("creating cache entry %s -> %s", key, cachePath);
+        if (!pathExists(dirOf(cachePath)))
+            createDirs(dirOf(cachePath));
+        writeFile(cachePath, sink.s);
+
+        debug("%s fetched with git-lfs", pointerFilePath);
+    } catch (const nlohmann::json::out_of_range & e) {
+        throw Error("bad json from /info/lfs/objects/batch: %s %s", obj, e.what());
+    }
+}
+
+} // namespace nix::lfs

src/libfetchers/git-lfs-fetch.hh

@@ -0,0 +1,43 @@
+#include "canon-path.hh"
+#include "serialise.hh"
+#include "url.hh"
+
+#include <git2/repository.h>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix::lfs {
+
+/**
+ * git-lfs pointer
+ * @see https://github.com/git-lfs/git-lfs/blob/2ef4108/docs/spec.md
+ */
+struct Pointer
+{
+    std::string oid; // git-lfs managed object id. you give this to the lfs server
+                     // for downloads
+    size_t size;     // in bytes
+};
+
+struct Fetch
+{
+    // Reference to the repository
+    const git_repository * repo;
+
+    // Git commit being fetched
+    git_oid rev;
+
+    // derived from git remote url
+    nix::ParsedURL url;
+
+    Fetch(git_repository * repo, git_oid rev);
+    bool shouldFetch(const CanonPath & path) const;
+    void fetch(
+        const std::string & content,
+        const CanonPath & pointerFilePath,
+        StringSink & sink,
+        std::function<void(uint64_t)> sizeCallback) const;
+    std::vector<nlohmann::json> fetchUrls(const std::vector<Pointer> & pointers) const;
+};
+
+} // namespace nix::lfs

src/libfetchers/git-utils.cc

@@ -1,4 +1,5 @@
 #include "git-utils.hh"
+#include "git-lfs-fetch.hh"
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
@@ -60,14 +61,6 @@ namespace nix {
 
 struct GitSourceAccessor;
 
-// Some wrapper types that ensure that the git_*_free functions get called.
-template<auto del>
-struct Deleter
-{
-    template <typename T>
-    void operator()(T * p) const { del(p); };
-};
-
 typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
 typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
 typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
@@ -85,20 +78,6 @@ typedef std::unique_ptr<git_odb, Deleter<git_odb_free>> ObjectDb;
 typedef std::unique_ptr<git_packbuilder, Deleter<git_packbuilder_free>> PackBuilder;
 typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
 
-// A helper to ensure that we don't leak objects returned by libgit2.
-template<typename T>
-struct Setter
-{
-    T & t;
-    typename T::pointer p = nullptr;
-
-    Setter(T & t) : t(t) { }
-
-    ~Setter() { if (p) t = T(p); }
-
-    operator typename T::pointer * () { return &p; }
-};
-
 Hash toHash(const git_oid & oid)
 {
     #ifdef GIT_EXPERIMENTAL_SHA256
@@ -506,12 +485,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     /**
      * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev);
+    ref<GitSourceAccessor> getRawAccessor(
+        const Hash & rev,
+        bool smudgeLfs = false);
 
     ref<SourceAccessor> getAccessor(
         const Hash & rev,
         bool exportIgnore,
-        std::string displayPrefix) override;
+        std::string displayPrefix,
+        bool smudgeLfs = false) override;
 
     ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
 
@@ -670,24 +652,40 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 /**
  * Raw git tree input accessor.
  */
+
 struct GitSourceAccessor : SourceAccessor
 {
     ref<GitRepoImpl> repo;
     Object root;
+    std::optional<lfs::Fetch> lfsFetch = std::nullopt;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
         : repo(repo_)
         , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get()))
     {
+        if (smudgeLfs)
+            lfsFetch = std::make_optional(lfs::Fetch(*repo, hashToOID(rev)));
     }
 
     std::string readBlob(const CanonPath & path, bool symlink)
     {
-        auto blob = getBlob(path, symlink);
+        const auto blob = getBlob(path, symlink);
 
-        auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+        if (lfsFetch) {
+            if (lfsFetch->shouldFetch(path)) {
+                StringSink s;
+                try {
+                    auto contents = std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+                    lfsFetch->fetch(contents, path, s, [&s](uint64_t size){ s.s.reserve(size); });
+                } catch (Error & e) {
+                    e.addTrace({}, "while smudging git-lfs file '%s'", path);
+                    throw;
+                }
+                return s.s;
+            }
+        }
 
-        return std::string(data);
+        return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
     }
 
     std::string readFile(const CanonPath & path) override
@@ -1191,19 +1189,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(
+    const Hash & rev,
+    bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev);
+    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
 }
 
 ref<SourceAccessor> GitRepoImpl::getAccessor(
     const Hash & rev,
     bool exportIgnore,
-    std::string displayPrefix)
+    std::string displayPrefix,
+    bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
     if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);

src/libfetchers/git-utils.hh

@@ -89,7 +89,8 @@ struct GitRepo
     virtual ref<SourceAccessor> getAccessor(
         const Hash & rev,
         bool exportIgnore,
-        std::string displayPrefix) = 0;
+        std::string displayPrefix,
+        bool smudgeLfs = false) = 0;
 
     virtual ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 
@@ -126,4 +127,26 @@ struct GitRepo
 
 ref<GitRepo> getTarballCache();
 
+// A helper to ensure that the `git_*_free` functions get called.
+template<auto del>
+struct Deleter
+{
+    template <typename T>
+    void operator()(T * p) const { del(p); };
+};
+
+// A helper to ensure that we don't leak objects returned by libgit2.
+template<typename T>
+struct Setter
+{
+    T & t;
+    typename T::pointer p = nullptr;
+
+    Setter(T & t) : t(t) { }
+
+    ~Setter() { if (p) t = T(p); }
+
+    operator typename T::pointer * () { return &p; }
+};
+
 }

src/libfetchers/git.cc

@@ -9,7 +9,6 @@
 #include "pathlocks.hh"
 #include "processes.hh"
 #include "git.hh"
-#include "mounted-source-accessor.hh"
 #include "git-utils.hh"
 #include "logging.hh"
 #include "finally.hh"
@@ -185,7 +184,7 @@ struct GitInputScheme : InputScheme
         for (auto & [name, value] : url.query) {
             if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
+            else if (name == "shallow" || name == "submodules" || name == "lfs" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -210,6 +209,7 @@ struct GitInputScheme : InputScheme
             "rev",
             "shallow",
             "submodules",
+            "lfs",
             "exportIgnore",
             "lastModified",
             "revCount",
@@ -262,6 +262,8 @@ struct GitInputScheme : InputScheme
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         if (getShallowAttr(input))
             url.query.insert_or_assign("shallow", "1");
+        if (getLfsAttr(input))
+            url.query.insert_or_assign("lfs", "1");
         if (getSubmodulesAttr(input))
             url.query.insert_or_assign("submodules", "1");
         if (maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false))
@@ -349,8 +351,7 @@ struct GitInputScheme : InputScheme
 
             if (commitMsg) {
                 // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
-                logger->pause();
-                Finally restoreLogger([]() { logger->resume(); });
+                auto suspension = logger->suspend();
                 runProgram("git", true,
                     { "-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
                     *commitMsg);
@@ -411,6 +412,11 @@ struct GitInputScheme : InputScheme
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getLfsAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "lfs").value_or(false);
+    }
+
     bool getExportIgnoreAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false);
@@ -678,7 +684,8 @@ struct GitInputScheme : InputScheme
         verifyCommit(input, repo);
 
         bool exportIgnore = getExportIgnoreAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»");
+        bool smudgeLfs = getLfsAttr(input);
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -698,6 +705,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
                 attrs.insert_or_assign("submodules", Explicit<bool>{ true });
+                attrs.insert_or_assign("lfs", Explicit<bool>{ smudgeLfs });
                 attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
                 auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
@@ -838,7 +846,7 @@ struct GitInputScheme : InputScheme
     {
         auto makeFingerprint = [&](const Hash & rev)
         {
-            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
+            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") + (getLfsAttr(input) ? ";l" : "");
         };
 
         if (auto rev = input.getRev())

src/libfetchers/github.cc

@@ -172,9 +172,30 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host) const
+    // Search for the longest possible match starting from the begining and ending at either the end or a path segment.
+    std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const override
     {
         auto tokens = settings.accessTokens.get();
+        std::string answer;
+        size_t answer_match_len = 0;
+        if(! url.empty()) {
+            for (auto & token : tokens) {
+                auto first = url.find(token.first);
+                if (
+                    first != std::string::npos
+                    && token.first.length() > answer_match_len
+                    && first == 0
+                    && url.substr(0,token.first.length()) == token.first
+                    && (url.length() == token.first.length() || url[token.first.length()] == '/')
+                    )
+                {
+                    answer = token.second;
+                    answer_match_len = token.first.length();
+                }
+            }
+            if (!answer.empty())
+                return answer;
+        }
         if (auto token = get(tokens, host))
             return *token;
         return {};
@@ -182,10 +203,22 @@ struct GitArchiveInputScheme : InputScheme
 
     Headers makeHeadersWithAuthTokens(
         const fetchers::Settings & settings,
-        const std::string & host) const
+        const std::string & host,
+        const Input & input) const
+    {
+        auto owner = getStrAttr(input.attrs, "owner");
+        auto repo = getStrAttr(input.attrs, "repo");
+        auto hostAndPath = fmt( "%s/%s/%s", host, owner, repo);
+        return makeHeadersWithAuthTokens(settings, host, hostAndPath);
+    }
+
+    Headers makeHeadersWithAuthTokens(
+        const fetchers::Settings & settings,
+        const std::string & host,
+        const std::string & hostAndPath) const
     {
         Headers headers;
-        auto accessToken = getAccessToken(settings, host);
+        auto accessToken = getAccessToken(settings, host, hostAndPath);
         if (accessToken) {
             auto hdr = accessHeaderFromToken(*accessToken);
             if (hdr)
@@ -361,7 +394,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             : "https://%s/api/v3/repos/%s/%s/commits/%s",
             host, getOwner(input), getRepo(input), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -378,7 +411,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         auto host = getHost(input);
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         // If we have no auth headers then we default to the public archive
         // urls so we do not run into rate limits.
@@ -435,7 +468,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -465,7 +498,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
         return DownloadUrl { url, headers };
     }
 
@@ -505,7 +538,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         auto base_url = fmt("https://%s/%s/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         std::string refUri;
         if (ref == "HEAD") {
@@ -552,7 +585,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
         return DownloadUrl { url, headers };
     }
 

src/libfetchers/meson.build

@@ -14,7 +14,7 @@ cxx = meson.get_compiler('cpp')
 
 subdir('nix-meson-build-support/deps-lists')
 
-configdata = configuration_data()
+configuration_data()
 
 deps_private_maybe_subproject = [
 ]
@@ -48,12 +48,12 @@ sources = files(
   'fetch-to-store.cc',
   'fetchers.cc',
   'filtering-source-accessor.cc',
+  'git-lfs-fetch.cc',
   'git-utils.cc',
   'git.cc',
   'github.cc',
   'indirect.cc',
   'mercurial.cc',
-  'mounted-source-accessor.cc',
   'path.cc',
   'registry.cc',
   'store-path-accessor.cc',
@@ -69,8 +69,8 @@ headers = files(
   'fetch-to-store.hh',
   'fetchers.hh',
   'filtering-source-accessor.hh',
+  'git-lfs-fetch.hh',
   'git-utils.hh',
-  'mounted-source-accessor.hh',
   'registry.hh',
   'store-path-accessor.hh',
   'tarball.hh',

src/libfetchers/mounted-source-accessor.hh

@@ -1,9 +0,0 @@
-#pragma once
-
-#include "source-accessor.hh"
-
-namespace nix {
-
-ref<SourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);
-
-}

src/libfetchers/path.cc

@@ -125,7 +125,7 @@ struct PathInputScheme : InputScheme
 
         auto absPath = getAbsPath(input);
 
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath));
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
 
         // FIXME: check whether access to 'path' is allowed.
         auto storePath = store->maybeParseStorePath(absPath.string());

src/libflake/flake/flake.cc

@@ -12,6 +12,7 @@
 #include "flake/settings.hh"
 #include "value-to-json.hh"
 #include "local-fs-store.hh"
+#include "fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -24,7 +25,7 @@ namespace flake {
 struct FetchedFlake
 {
     FlakeRef lockedRef;
-    StorePath storePath;
+    ref<SourceAccessor> accessor;
 };
 
 typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
@@ -40,7 +41,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
     return i->second;
 }
 
-static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
+static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     EvalState & state,
     const FlakeRef & originalRef,
     bool useRegistries,
@@ -51,8 +52,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
 
     if (!fetched) {
         if (originalRef.input.isDirect()) {
-            auto [storePath, lockedRef] = originalRef.fetchTree(state.store);
-            fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
+            auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
+            fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
         } else {
             if (useRegistries) {
                 resolvedRef = originalRef.resolve(
@@ -64,8 +65,8 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
                     });
                 fetched = lookupInFlakeCache(flakeCache, originalRef);
                 if (!fetched) {
-                    auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store);
-                    fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath});
+                    auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
+                    fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
                 }
                 flakeCache.insert_or_assign(resolvedRef, *fetched);
             }
@@ -76,14 +77,27 @@ static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
         flakeCache.insert_or_assign(originalRef, *fetched);
     }
 
-    debug("got tree '%s' from '%s'",
-        state.store->printStorePath(fetched->storePath), fetched->lockedRef);
+    debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef);
 
-    state.allowPath(fetched->storePath);
+    return {fetched->accessor, resolvedRef, fetched->lockedRef};
+}
 
-    assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store));
+static StorePath copyInputToStore(
+    EvalState & state,
+    fetchers::Input & input,
+    const fetchers::Input & originalInput,
+    ref<SourceAccessor> accessor)
+{
+    auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
 
-    return {fetched->storePath, resolvedRef, fetched->lockedRef};
+    state.allowPath(storePath);
+
+    auto narHash = state.store->queryPathInfo(storePath)->narHash;
+    input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+
+    assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
+
+    return storePath;
 }
 
 static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@@ -101,19 +115,54 @@ static void expectType(EvalState & state, ValueType type,
             showType(type), showType(value.type()), state.positions[pos]);
 }
 
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
     EvalState & state,
     Value * value,
     const PosIdx pos,
-    const InputPath & lockRootPath,
-    const SourcePath & flakeDir);
+    const InputAttrPath & lockRootAttrPath,
+    const SourcePath & flakeDir,
+    bool allowSelf);
+
+static void parseFlakeInputAttr(
+    EvalState & state,
+    const Attr & attr,
+    fetchers::Attrs & attrs)
+{
+    // Allow selecting a subset of enum values
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wswitch-enum"
+    switch (attr.value->type()) {
+        case nString:
+            attrs.emplace(state.symbols[attr.name], attr.value->c_str());
+            break;
+        case nBool:
+            attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
+            break;
+        case nInt: {
+            auto intValue = attr.value->integer().value;
+            if (intValue < 0)
+                state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
+            attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
+            break;
+        }
+        default:
+            if (attr.name == state.symbols.create("publicKeys")) {
+                experimentalFeatureSettings.require(Xp::VerifiedFetches);
+                NixStringContext emptyContext = {};
+                attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, attr.pos, emptyContext).dump());
+            } else
+                state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+                    state.symbols[attr.name], showType(*attr.value)).debugThrow();
+    }
+    #pragma GCC diagnostic pop
+}
 
 static FlakeInput parseFlakeInput(
     EvalState & state,
     std::string_view inputName,
     Value * value,
     const PosIdx pos,
-    const InputPath & lockRootPath,
+    const InputAttrPath & lockRootAttrPath,
     const SourcePath & flakeDir)
 {
     expectType(state, nAttrs, *value, pos);
@@ -137,7 +186,7 @@ static FlakeInput parseFlakeInput(
                 else if (attr.value->type() == nPath) {
                     auto path = attr.value->path();
                     if (path.accessor != flakeDir.accessor)
-                        throw Error("input path '%s' at %s must be in the same source tree as %s",
+                        throw Error("input attribute path '%s' at %s must be in the same source tree as %s",
                             path, state.positions[attr.pos], flakeDir);
                     url = "path:" + flakeDir.path.makeRelative(path.path);
                 }
@@ -149,44 +198,14 @@ static FlakeInput parseFlakeInput(
                 expectType(state, nBool, *attr.value, attr.pos);
                 input.isFlake = attr.value->boolean();
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath, flakeDir);
+                input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first;
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, attr.pos);
-                auto follows(parseInputPath(attr.value->c_str()));
-                follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end());
+                auto follows(parseInputAttrPath(attr.value->c_str()));
+                follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end());
                 input.follows = follows;
-            } else {
-                // Allow selecting a subset of enum values
-                #pragma GCC diagnostic push
-                #pragma GCC diagnostic ignored "-Wswitch-enum"
-                switch (attr.value->type()) {
-                    case nString:
-                        attrs.emplace(state.symbols[attr.name], attr.value->c_str());
-                        break;
-                    case nBool:
-                        attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean() });
-                        break;
-                    case nInt: {
-                        auto intValue = attr.value->integer().value;
-
-                        if (intValue < 0) {
-                            state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
-                        }
-
-                        attrs.emplace(state.symbols[attr.name], uint64_t(intValue));
-                        break;
-                    }
-                    default:
-                        if (attr.name == state.symbols.create("publicKeys")) {
-                            experimentalFeatureSettings.require(Xp::VerifiedFetches);
-                            NixStringContext emptyContext = {};
-                            attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump());
-                        } else
-                            state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
-                                state.symbols[attr.name], showType(*attr.value)).debugThrow();
-                }
-                #pragma GCC diagnostic pop
-            }
+            } else
+                parseFlakeInputAttr(state, attr, attrs);
         } catch (Error & e) {
             e.addTrace(
                 state.positions[attr.pos],
@@ -216,28 +235,39 @@ static FlakeInput parseFlakeInput(
     return input;
 }
 
-static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInputs(
     EvalState & state,
     Value * value,
     const PosIdx pos,
-    const InputPath & lockRootPath,
-    const SourcePath & flakeDir)
+    const InputAttrPath & lockRootAttrPath,
+    const SourcePath & flakeDir,
+    bool allowSelf)
 {
     std::map<FlakeId, FlakeInput> inputs;
+    fetchers::Attrs selfAttrs;
 
     expectType(state, nAttrs, *value, pos);
 
     for (auto & inputAttr : *value->attrs()) {
-        inputs.emplace(state.symbols[inputAttr.name],
-            parseFlakeInput(state,
-                state.symbols[inputAttr.name],
-                inputAttr.value,
-                inputAttr.pos,
-                lockRootPath,
-                flakeDir));
+        auto inputName = state.symbols[inputAttr.name];
+        if (inputName == "self") {
+            if (!allowSelf)
+                throw Error("'self' input attribute not allowed at %s", state.positions[inputAttr.pos]);
+            expectType(state, nAttrs, *inputAttr.value, inputAttr.pos);
+            for (auto & attr : *inputAttr.value->attrs())
+                parseFlakeInputAttr(state, attr, selfAttrs);
+        } else {
+            inputs.emplace(inputName,
+                parseFlakeInput(state,
+                    inputName,
+                    inputAttr.value,
+                    inputAttr.pos,
+                    lockRootAttrPath,
+                    flakeDir));
+        }
     }
 
-    return inputs;
+    return {inputs, selfAttrs};
 }
 
 static Flake readFlake(
@@ -246,7 +276,7 @@ static Flake readFlake(
     const FlakeRef & resolvedRef,
     const FlakeRef & lockedRef,
     const SourcePath & rootDir,
-    const InputPath & lockRootPath)
+    const InputAttrPath & lockRootAttrPath)
 {
     auto flakeDir = rootDir / CanonPath(resolvedRef.subdir);
     auto flakePath = flakeDir / "flake.nix";
@@ -269,8 +299,11 @@ static Flake readFlake(
 
     auto sInputs = state.symbols.create("inputs");
 
-    if (auto inputs = vInfo.attrs()->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath, flakeDir);
+    if (auto inputs = vInfo.attrs()->get(sInputs)) {
+        auto [flakeInputs, selfAttrs] = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootAttrPath, flakeDir, true);
+        flake.inputs = std::move(flakeInputs);
+        flake.selfAttrs = std::move(selfAttrs);
+    }
 
     auto sOutputs = state.symbols.create("outputs");
 
@@ -301,10 +334,10 @@ static Flake readFlake(
                     state.symbols[setting.name],
                     std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
             else if (setting.value->type() == nPath) {
-                NixStringContext emptyContext = {};
+                auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
-                    state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned());
+                    state.store->printStorePath(storePath));
             }
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(
@@ -342,17 +375,55 @@ static Flake readFlake(
     return flake;
 }
 
+static FlakeRef applySelfAttrs(
+    const FlakeRef & ref,
+    const Flake & flake)
+{
+    auto newRef(ref);
+
+    std::set<std::string> allowedAttrs{"submodules", "lfs"};
+
+    for (auto & attr : flake.selfAttrs) {
+        if (!allowedAttrs.contains(attr.first))
+            throw Error("flake 'self' attribute '%s' is not supported", attr.first);
+        newRef.input.attrs.insert_or_assign(attr.first, attr.second);
+    }
+
+    return newRef;
+}
+
 static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
     bool useRegistries,
     FlakeCache & flakeCache,
-    const InputPath & lockRootPath)
+    const InputAttrPath & lockRootAttrPath)
 {
-    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+    // Fetch a lazy tree first.
+    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, useRegistries, flakeCache);
 
-    return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath);
+    // Parse/eval flake.nix to get at the input.self attributes.
+    auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath);
+
+    // Re-fetch the tree if necessary.
+    auto newLockedRef = applySelfAttrs(lockedRef, flake);
+
+    if (lockedRef != newLockedRef) {
+        debug("refetching input '%s' due to self attribute", newLockedRef);
+        // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
+        newLockedRef.input.attrs.erase("narHash");
+        auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
+            state, newLockedRef, false, flakeCache);
+        accessor = accessor2;
+        lockedRef = lockedRef2;
+    }
+
+    // Copy the tree to the store.
+    auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor);
+
+    // Re-parse flake.nix from the store.
+    return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
 }
 
 Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries)
@@ -405,12 +476,12 @@ LockedFlake lockFlake(
         {
             FlakeInput input;
             SourcePath sourcePath;
-            std::optional<InputPath> parentInputPath; // FIXME: rename to inputPathPrefix?
+            std::optional<InputAttrPath> parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix?
         };
 
-        std::map<InputPath, OverrideTarget> overrides;
-        std::set<InputPath> explicitCliOverrides;
-        std::set<InputPath> overridesUsed, updatesUsed;
+        std::map<InputAttrPath, OverrideTarget> overrides;
+        std::set<InputAttrPath> explicitCliOverrides;
+        std::set<InputAttrPath> overridesUsed, updatesUsed;
         std::map<ref<Node>, SourcePath> nodePaths;
 
         for (auto & i : lockFlags.inputOverrides) {
@@ -434,9 +505,9 @@ LockedFlake lockFlake(
         std::function<void(
             const FlakeInputs & flakeInputs,
             ref<Node> node,
-            const InputPath & inputPathPrefix,
+            const InputAttrPath & inputAttrPathPrefix,
             std::shared_ptr<const Node> oldNode,
-            const InputPath & followsPrefix,
+            const InputAttrPath & followsPrefix,
             const SourcePath & sourcePath,
             bool trustLock)>
             computeLocks;
@@ -448,7 +519,7 @@ LockedFlake lockFlake(
             /* The node whose locks are to be updated.*/
             ref<Node> node,
             /* The path to this node in the lock file graph. */
-            const InputPath & inputPathPrefix,
+            const InputAttrPath & inputAttrPathPrefix,
             /* The old node, if any, from which locks can be
                copied. */
             std::shared_ptr<const Node> oldNode,
@@ -456,59 +527,59 @@ LockedFlake lockFlake(
                interpreted. When a node is initially locked, it's
                relative to the node's flake; when it's already locked,
                it's relative to the root of the lock file. */
-            const InputPath & followsPrefix,
+            const InputAttrPath & followsPrefix,
             /* The source path of this node's flake. */
             const SourcePath & sourcePath,
             bool trustLock)
         {
-            debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+            debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix));
 
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
             for (auto & [id, input] : flakeInputs) {
                 for (auto & [idOverride, inputOverride] : input.overrides) {
-                    auto inputPath(inputPathPrefix);
-                    inputPath.push_back(id);
-                    inputPath.push_back(idOverride);
-                    overrides.emplace(inputPath,
+                    auto inputAttrPath(inputAttrPathPrefix);
+                    inputAttrPath.push_back(id);
+                    inputAttrPath.push_back(idOverride);
+                    overrides.emplace(inputAttrPath,
                         OverrideTarget {
                             .input = inputOverride,
                             .sourcePath = sourcePath,
-                            .parentInputPath = inputPathPrefix
+                            .parentInputAttrPath = inputAttrPathPrefix
                         });
                 }
             }
 
             /* Check whether this input has overrides for a
                non-existent input. */
-            for (auto [inputPath, inputOverride] : overrides) {
-                auto inputPath2(inputPath);
-                auto follow = inputPath2.back();
-                inputPath2.pop_back();
-                if (inputPath2 == inputPathPrefix && !flakeInputs.count(follow))
+            for (auto [inputAttrPath, inputOverride] : overrides) {
+                auto inputAttrPath2(inputAttrPath);
+                auto follow = inputAttrPath2.back();
+                inputAttrPath2.pop_back();
+                if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow))
                     warn(
                         "input '%s' has an override for a non-existent input '%s'",
-                        printInputPath(inputPathPrefix), follow);
+                        printInputAttrPath(inputAttrPathPrefix), follow);
             }
 
             /* Go over the flake inputs, resolve/fetch them if
                necessary (i.e. if they're new or the flakeref changed
                from what's in the lock file). */
             for (auto & [id, input2] : flakeInputs) {
-                auto inputPath(inputPathPrefix);
-                inputPath.push_back(id);
-                auto inputPathS = printInputPath(inputPath);
-                debug("computing input '%s'", inputPathS);
+                auto inputAttrPath(inputAttrPathPrefix);
+                inputAttrPath.push_back(id);
+                auto inputAttrPathS = printInputAttrPath(inputAttrPath);
+                debug("computing input '%s'", inputAttrPathS);
 
                 try {
 
                     /* Do we have an override for this input from one of the
                        ancestors? */
-                    auto i = overrides.find(inputPath);
+                    auto i = overrides.find(inputAttrPath);
                     bool hasOverride = i != overrides.end();
-                    bool hasCliOverride = explicitCliOverrides.contains(inputPath);
+                    bool hasCliOverride = explicitCliOverrides.contains(inputAttrPath);
                     if (hasOverride)
-                        overridesUsed.insert(inputPath);
+                        overridesUsed.insert(inputAttrPath);
                     auto input = hasOverride ? i->second.input : input2;
 
                     /* Resolve relative 'path:' inputs relative to
@@ -523,11 +594,11 @@ LockedFlake lockFlake(
                     /* Resolve 'follows' later (since it may refer to an input
                        path we haven't processed yet. */
                     if (input.follows) {
-                        InputPath target;
+                        InputAttrPath target;
 
                         target.insert(target.end(), input.follows->begin(), input.follows->end());
 
-                        debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+                        debug("input '%s' follows '%s'", inputAttrPathS, printInputAttrPath(target));
                         node->inputs.insert_or_assign(id, target);
                         continue;
                     }
@@ -536,7 +607,7 @@ LockedFlake lockFlake(
 
                     auto overridenParentPath =
                         input.ref->input.isRelative()
-                        ? std::optional<InputPath>(hasOverride ? i->second.parentInputPath : inputPathPrefix)
+                        ? std::optional<InputAttrPath>(hasOverride ? i->second.parentInputAttrPath : inputAttrPathPrefix)
                         : std::nullopt;
 
                     auto resolveRelativePath = [&]() -> std::optional<SourcePath>
@@ -555,9 +626,9 @@ LockedFlake lockFlake(
                     auto getInputFlake = [&](const FlakeRef & ref)
                     {
                         if (auto resolvedPath = resolveRelativePath()) {
-                            return readFlake(state, ref, ref, ref, *resolvedPath, inputPath);
+                            return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath);
                         } else {
-                            return getFlake(state, ref, useRegistries, flakeCache, inputPath);
+                            return getFlake(state, ref, useRegistries, flakeCache, inputAttrPath);
                         }
                     };
 
@@ -565,19 +636,19 @@ LockedFlake lockFlake(
                        And the input is not in updateInputs? */
                     std::shared_ptr<LockedNode> oldLock;
 
-                    updatesUsed.insert(inputPath);
+                    updatesUsed.insert(inputAttrPath);
 
-                    if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+                    if (oldNode && !lockFlags.inputUpdates.count(inputAttrPath))
                         if (auto oldLock2 = get(oldNode->inputs, id))
                             if (auto oldLock3 = std::get_if<0>(&*oldLock2))
                                 oldLock = *oldLock3;
 
                     if (oldLock
                         && oldLock->originalRef == *input.ref
-                        && oldLock->parentPath == overridenParentPath
+                        && oldLock->parentInputAttrPath == overridenParentPath
                         && !hasCliOverride)
                     {
-                        debug("keeping existing input '%s'", inputPathS);
+                        debug("keeping existing input '%s'", inputAttrPathS);
 
                         /* Copy the input from the old lock since its flakeref
                            didn't change and there is no override from a
@@ -586,18 +657,18 @@ LockedFlake lockFlake(
                             oldLock->lockedRef,
                             oldLock->originalRef,
                             oldLock->isFlake,
-                            oldLock->parentPath);
+                            oldLock->parentInputAttrPath);
 
                         node->inputs.insert_or_assign(id, childNode);
 
                         /* If we have this input in updateInputs, then we
                            must fetch the flake to update it. */
-                        auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+                        auto lb = lockFlags.inputUpdates.lower_bound(inputAttrPath);
 
                         auto mustRefetch =
                             lb != lockFlags.inputUpdates.end()
-                            && lb->size() > inputPath.size()
-                            && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+                            && lb->size() > inputAttrPath.size()
+                            && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->begin());
 
                         FlakeInputs fakeInputs;
 
@@ -616,7 +687,7 @@ LockedFlake lockFlake(
                                     if (!trustLock) {
                                         // It is possible that the flake has changed,
                                         // so we must confirm all the follows that are in the lock file are also in the flake.
-                                        auto overridePath(inputPath);
+                                        auto overridePath(inputAttrPath);
                                         overridePath.push_back(i.first);
                                         auto o = overrides.find(overridePath);
                                         // If the override disappeared, we have to refetch the flake,
@@ -640,21 +711,21 @@ LockedFlake lockFlake(
                         if (mustRefetch) {
                             auto inputFlake = getInputFlake(oldLock->lockedRef);
                             nodePaths.emplace(childNode, inputFlake.path.parent());
-                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, followsPrefix,
+                            computeLocks(inputFlake.inputs, childNode, inputAttrPath, oldLock, followsPrefix,
                                 inputFlake.path, false);
                         } else {
-                            computeLocks(fakeInputs, childNode, inputPath, oldLock, followsPrefix, sourcePath, true);
+                            computeLocks(fakeInputs, childNode, inputAttrPath, oldLock, followsPrefix, sourcePath, true);
                         }
 
                     } else {
                         /* We need to create a new lock file entry. So fetch
                            this input. */
-                        debug("creating new input '%s'", inputPathS);
+                        debug("creating new input '%s'", inputAttrPathS);
 
                         if (!lockFlags.allowUnlocked
                             && !input.ref->input.isLocked()
                             && !input.ref->input.isRelative())
-                            throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS);
+                            throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS);
 
                         /* Note: in case of an --override-input, we use
                             the *original* ref (input2.ref) for the
@@ -663,7 +734,7 @@ LockedFlake lockFlake(
                             nuked the next time we update the lock
                             file. That is, overrides are sticky unless you
                             use --no-write-lock-file. */
-                        auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
+                        auto ref = (input2.ref && explicitCliOverrides.contains(inputAttrPath)) ? *input2.ref : *input.ref;
 
                         if (input.isFlake) {
                             auto inputFlake = getInputFlake(*input.ref);
@@ -689,11 +760,11 @@ LockedFlake lockFlake(
                                own lock file. */
                             nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(
-                                inputFlake.inputs, childNode, inputPath,
+                                inputFlake.inputs, childNode, inputAttrPath,
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(),
-                                oldLock ? followsPrefix : inputPath,
+                                oldLock ? followsPrefix : inputAttrPath,
                                 inputFlake.path,
                                 false);
                         }
@@ -705,9 +776,13 @@ LockedFlake lockFlake(
                                 if (auto resolvedPath = resolveRelativePath()) {
                                     return {*resolvedPath, *input.ref};
                                 } else {
-                                    auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                                    auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
                                         state, *input.ref, useRegistries, flakeCache);
-                                    return {state.rootPath(state.store->toRealPath(storePath)), lockedRef};
+
+                                    // FIXME: allow input to be lazy.
+                                    auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
+
+                                    return {state.storePath(storePath), lockedRef};
                                 }
                             }();
 
@@ -720,7 +795,7 @@ LockedFlake lockFlake(
                     }
 
                 } catch (Error & e) {
-                    e.addTrace({}, "while updating the flake input '%s'", inputPathS);
+                    e.addTrace({}, "while updating the flake input '%s'", inputAttrPathS);
                     throw;
                 }
             }
@@ -740,11 +815,11 @@ LockedFlake lockFlake(
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
                 warn("the flag '--override-input %s %s' does not match any input",
-                    printInputPath(i.first), i.second);
+                    printInputAttrPath(i.first), i.second);
 
         for (auto & i : lockFlags.inputUpdates)
             if (!updatesUsed.count(i))
-                warn("'%s' does not match any input of this flake", printInputPath(i));
+                warn("'%s' does not match any input of this flake", printInputAttrPath(i));
 
         /* Check 'follows' inputs. */
         newLockFile.check();
@@ -844,21 +919,6 @@ LockedFlake lockFlake(
     }
 }
 
-std::pair<StorePath, Path> sourcePathToStorePath(
-    ref<Store> store,
-    const SourcePath & _path)
-{
-    auto path = _path.path.abs();
-
-    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
-        auto realStoreDir = store2->getRealStoreDir();
-        if (isInDir(path, realStoreDir))
-            path = store2->storeDir + path.substr(realStoreDir.size());
-    }
-
-    return store->toStorePath(path);
-}
-
 void callFlake(EvalState & state,
     const LockedFlake & lockedFlake,
     Value & vRes)
@@ -874,7 +934,7 @@ void callFlake(EvalState & state,
 
         auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
 
-        auto [storePath, subdir] = sourcePathToStorePath(state.store, sourcePath);
+        auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs());
 
         emitTreeAttrs(
             state,

src/libflake/flake/flake.hh

@@ -57,7 +57,7 @@ struct FlakeInput
      * false = (fetched) static source path
      */
     bool isFlake = true;
-    std::optional<InputPath> follows;
+    std::optional<InputAttrPath> follows;
     FlakeInputs overrides;
 };
 
@@ -79,24 +79,37 @@ struct Flake
      * The original flake specification (by the user)
      */
     FlakeRef originalRef;
+
     /**
      * registry references and caching resolved to the specific underlying flake
      */
     FlakeRef resolvedRef;
+
     /**
      * the specific local store result of invoking the fetcher
      */
     FlakeRef lockedRef;
+
     /**
      * The path of `flake.nix`.
      */
     SourcePath path;
+
     /**
-     * pretend that 'lockedRef' is dirty
+     * Pretend that `lockedRef` is dirty.
      */
     bool forceDirty = false;
+
     std::optional<std::string> description;
+
     FlakeInputs inputs;
+
+    /**
+     * Attributes to be retroactively applied to the `self` input
+     * (such as `submodules = true`).
+     */
+    fetchers::Attrs selfAttrs;
+
     /**
      * 'nixConfig' attribute
      */
@@ -201,13 +214,13 @@ struct LockFlags
     /**
      * Flake inputs to be overridden.
      */
-    std::map<InputPath, FlakeRef> inputOverrides;
+    std::map<InputAttrPath, FlakeRef> inputOverrides;
 
     /**
      * Flake inputs to be updated. This means that any existing lock
      * for those inputs will be ignored.
      */
-    std::set<InputPath> inputUpdates;
+    std::set<InputAttrPath> inputUpdates;
 };
 
 LockedFlake lockFlake(
@@ -221,16 +234,6 @@ void callFlake(
     const LockedFlake & lockedFlake,
     Value & v);
 
-/**
- * Map a `SourcePath` to the corresponding store path. This is a
- * temporary hack to support chroot stores while we don't have full
- * lazy trees. FIXME: Remove this once we can pass a sourcePath rather
- * than a storePath to call-flake.nix.
- */
-std::pair<StorePath, Path> sourcePathToStorePath(
-    ref<Store> store,
-    const SourcePath & path);
-
 }
 
 void emitTreeAttrs(

src/libflake/flake/flakeref.cc

@@ -107,7 +107,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
            to 'baseDir'). If so, search upward to the root of the
            repo (i.e. the directory containing .git). */
 
-        path = absPath(path, baseDir);
+        path = absPath(path, baseDir, true);
 
         if (isFlake) {
 
@@ -283,10 +283,10 @@ FlakeRef FlakeRef::fromAttrs(
         fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
 }
 
-std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
+std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
 {
-    auto [storePath, lockedInput] = input.fetchToStore(store);
-    return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
+    auto [accessor, lockedInput] = input.getAccessor(store);
+    return {accessor, FlakeRef(std::move(lockedInput), subdir)};
 }
 
 std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(

src/libflake/flake/flakeref.hh

@@ -71,7 +71,7 @@ struct FlakeRef
         const fetchers::Settings & fetchSettings,
         const fetchers::Attrs & attrs);
 
-    std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
+    std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
 };
 
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

src/libflake/flake/lockfile.cc

@@ -1,7 +1,10 @@
 #include <unordered_set>
 
+#include "fetch-settings.hh"
+#include "flake/settings.hh"
 #include "lockfile.hh"
 #include "store-api.hh"
+#include "strings.hh"
 
 #include <algorithm>
 #include <iomanip>
@@ -9,8 +12,6 @@
 #include <iterator>
 #include <nlohmann/json.hpp>
 
-#include "strings.hh"
-#include "flake/settings.hh"
 
 namespace nix::flake {
 
@@ -43,11 +44,18 @@ LockedNode::LockedNode(
     : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info"
     , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr))
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
-    , parentPath(json.find("parent") != json.end() ? (std::optional<InputPath>) json["parent"] : std::nullopt)
+    , parentInputAttrPath(json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt)
 {
-    if (!lockedRef.input.isConsideredLocked(fetchSettings) && !lockedRef.input.isRelative())
-        throw Error("Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.",
-            fetchers::attrsToJSON(lockedRef.input.toAttrs()));
+    if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
+        if (lockedRef.input.getNarHash())
+            warn(
+                "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
+                "This is deprecated since such inputs are verifiable but may not be reproducible.",
+                lockedRef.to_string());
+        else
+            throw Error("Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.",
+                fetchers::attrsToJSON(lockedRef.input.toAttrs()));
+    }
 
     // For backward compatibility, lock file entries are implicitly final.
     assert(!lockedRef.input.attrs.contains("__final"));
@@ -59,7 +67,7 @@ StorePath LockedNode::computeStorePath(Store & store) const
     return lockedRef.input.computeStorePath(store);
 }
 
-static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputPath & path, std::vector<InputPath> & visited)
+static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputAttrPath & path, std::vector<InputAttrPath> & visited)
 {
     auto pos = root;
 
@@ -67,8 +75,8 @@ static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputPath & pa
 
     if (found != visited.end()) {
         std::vector<std::string> cycle;
-        std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath);
-        cycle.push_back(printInputPath(path));
+        std::transform(found, visited.cend(), std::back_inserter(cycle), printInputAttrPath);
+        cycle.push_back(printInputAttrPath(path));
         throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle));
     }
     visited.push_back(path);
@@ -90,9 +98,9 @@ static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputPath & pa
     return pos;
 }
 
-std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
+std::shared_ptr<Node> LockFile::findInput(const InputAttrPath & path)
 {
-    std::vector<InputPath> visited;
+    std::vector<InputAttrPath> visited;
     return doFind(root, path, visited);
 }
 
@@ -115,7 +123,7 @@ LockFile::LockFile(
         if (jsonNode.find("inputs") == jsonNode.end()) return;
         for (auto & i : jsonNode["inputs"].items()) {
             if (i.value().is_array()) { // FIXME: remove, obsolete
-                InputPath path;
+                InputAttrPath path;
                 for (auto & j : i.value())
                     path.push_back(j);
                 node.inputs.insert_or_assign(i.key(), path);
@@ -203,8 +211,8 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
             n["locked"].erase("__final");
             if (!lockedNode->isFlake)
                 n["flake"] = false;
-            if (lockedNode->parentPath)
-                n["parent"] = *lockedNode->parentPath;
+            if (lockedNode->parentInputAttrPath)
+                n["parent"] = *lockedNode->parentInputAttrPath;
         }
 
         nodes[key] = std::move(n);
@@ -248,11 +256,20 @@ std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSet
 
     visit(root);
 
+    /* Return whether the input is either locked, or, if
+       `allow-dirty-locks` is enabled, it has a NAR hash. In the
+       latter case, we can verify the input but we may not be able to
+       fetch it from anywhere. */
+    auto isConsideredLocked = [&](const fetchers::Input & input)
+    {
+        return input.isLocked() || (fetchSettings.allowDirtyLocks && input.getNarHash());
+    };
+
     for (auto & i : nodes) {
         if (i == ref<const Node>(root)) continue;
         auto node = i.dynamic_pointer_cast<const LockedNode>();
         if (node
-            && (!node->lockedRef.input.isConsideredLocked(fetchSettings)
+            && (!isConsideredLocked(node->lockedRef.input)
                 || !node->lockedRef.input.isFinal())
             && !node->lockedRef.input.isRelative())
             return node->lockedRef;
@@ -267,36 +284,36 @@ bool LockFile::operator ==(const LockFile & other) const
     return toJSON().first == other.toJSON().first;
 }
 
-InputPath parseInputPath(std::string_view s)
+InputAttrPath parseInputAttrPath(std::string_view s)
 {
-    InputPath path;
+    InputAttrPath path;
 
     for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
         if (!std::regex_match(elem, flakeIdRegex))
-            throw UsageError("invalid flake input path element '%s'", elem);
+            throw UsageError("invalid flake input attribute path element '%s'", elem);
         path.push_back(elem);
     }
 
     return path;
 }
 
-std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
+std::map<InputAttrPath, Node::Edge> LockFile::getAllInputs() const
 {
     std::set<ref<Node>> done;
-    std::map<InputPath, Node::Edge> res;
+    std::map<InputAttrPath, Node::Edge> res;
 
-    std::function<void(const InputPath & prefix, ref<Node> node)> recurse;
+    std::function<void(const InputAttrPath & prefix, ref<Node> node)> recurse;
 
-    recurse = [&](const InputPath & prefix, ref<Node> node)
+    recurse = [&](const InputAttrPath & prefix, ref<Node> node)
     {
         if (!done.insert(node).second) return;
 
         for (auto &[id, input] : node->inputs) {
-            auto inputPath(prefix);
-            inputPath.push_back(id);
-            res.emplace(inputPath, input);
+            auto inputAttrPath(prefix);
+            inputAttrPath.push_back(id);
+            res.emplace(inputAttrPath, input);
             if (auto child = std::get_if<0>(&input))
-                recurse(inputPath, *child);
+                recurse(inputAttrPath, *child);
         }
     };
 
@@ -320,7 +337,7 @@ std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
     if (auto node = std::get_if<0>(&edge))
         stream << describe((*node)->lockedRef);
     else if (auto follows = std::get_if<1>(&edge))
-        stream << fmt("follows '%s'", printInputPath(*follows));
+        stream << fmt("follows '%s'", printInputAttrPath(*follows));
     return stream;
 }
 
@@ -347,15 +364,15 @@ std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
     while (i != oldFlat.end() || j != newFlat.end()) {
         if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
             res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n    %s\n",
-                printInputPath(j->first), j->second);
+                printInputAttrPath(j->first), j->second);
             ++j;
         } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
-            res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first));
+            res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputAttrPath(i->first));
             ++i;
         } else {
             if (!equals(i->second, j->second)) {
                 res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n    %s\n  → %s\n",
-                    printInputPath(i->first),
+                    printInputAttrPath(i->first),
                     i->second,
                     j->second);
             }
@@ -371,19 +388,19 @@ void LockFile::check()
 {
     auto inputs = getAllInputs();
 
-    for (auto & [inputPath, input] : inputs) {
+    for (auto & [inputAttrPath, input] : inputs) {
         if (auto follows = std::get_if<1>(&input)) {
             if (!follows->empty() && !findInput(*follows))
                 throw Error("input '%s' follows a non-existent input '%s'",
-                    printInputPath(inputPath),
-                    printInputPath(*follows));
+                    printInputAttrPath(inputAttrPath),
+                    printInputAttrPath(*follows));
         }
     }
 }
 
 void check();
 
-std::string printInputPath(const InputPath & path)
+std::string printInputAttrPath(const InputAttrPath & path)
 {
     return concatStringsSep("/", path);
 }

src/libflake/flake/lockfile.hh

@@ -12,7 +12,7 @@ class StorePath;
 
 namespace nix::flake {
 
-typedef std::vector<FlakeId> InputPath;
+typedef std::vector<FlakeId> InputAttrPath;
 
 struct LockedNode;
 
@@ -23,7 +23,7 @@ struct LockedNode;
  */
 struct Node : std::enable_shared_from_this<Node>
 {
-    typedef std::variant<ref<LockedNode>, InputPath> Edge;
+    typedef std::variant<ref<LockedNode>, InputAttrPath> Edge;
 
     std::map<FlakeId, Edge> inputs;
 
@@ -40,17 +40,17 @@ struct LockedNode : Node
 
     /* The node relative to which relative source paths
        (e.g. 'path:../foo') are interpreted. */
-    std::optional<InputPath> parentPath;
+    std::optional<InputAttrPath> parentInputAttrPath;
 
     LockedNode(
         const FlakeRef & lockedRef,
         const FlakeRef & originalRef,
         bool isFlake = true,
-        std::optional<InputPath> parentPath = {})
-        : lockedRef(lockedRef)
-        , originalRef(originalRef)
+        std::optional<InputAttrPath> parentInputAttrPath = {})
+        : lockedRef(std::move(lockedRef))
+        , originalRef(std::move(originalRef))
         , isFlake(isFlake)
-        , parentPath(parentPath)
+        , parentInputAttrPath(std::move(parentInputAttrPath))
     { }
 
     LockedNode(
@@ -83,9 +83,9 @@ struct LockFile
 
     bool operator ==(const LockFile & other) const;
 
-    std::shared_ptr<Node> findInput(const InputPath & path);
+    std::shared_ptr<Node> findInput(const InputAttrPath & path);
 
-    std::map<InputPath, Node::Edge> getAllInputs() const;
+    std::map<InputAttrPath, Node::Edge> getAllInputs() const;
 
     static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
 
@@ -97,8 +97,8 @@ struct LockFile
 
 std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
 
-InputPath parseInputPath(std::string_view s);
+InputAttrPath parseInputAttrPath(std::string_view s);
 
-std::string printInputPath(const InputPath & path);
+std::string printInputAttrPath(const InputAttrPath & path);
 
 }

src/libmain/loggers.cc

@@ -6,7 +6,8 @@ namespace nix {
 
 LogFormat defaultLogFormat = LogFormat::raw;
 
-LogFormat parseLogFormat(const std::string & logFormatStr) {
+LogFormat parseLogFormat(const std::string & logFormatStr)
+{
     if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
         return LogFormat::raw;
     else if (logFormatStr == "raw-with-logs")
@@ -20,14 +21,15 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
     throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
 }
 
-Logger * makeDefaultLogger() {
+std::unique_ptr<Logger> makeDefaultLogger()
+{
     switch (defaultLogFormat) {
     case LogFormat::raw:
         return makeSimpleLogger(false);
     case LogFormat::rawWithLogs:
         return makeSimpleLogger(true);
     case LogFormat::internalJSON:
-        return makeJSONLogger(*makeSimpleLogger(true));
+        return makeJSONLogger(getStandardError());
     case LogFormat::bar:
         return makeProgressBar();
     case LogFormat::barWithLogs: {
@@ -40,16 +42,14 @@ Logger * makeDefaultLogger() {
     }
 }
 
-void setLogFormat(const std::string & logFormatStr) {
+void setLogFormat(const std::string & logFormatStr)
+{
     setLogFormat(parseLogFormat(logFormatStr));
 }
 
-void setLogFormat(const LogFormat & logFormat) {
+void setLogFormat(const LogFormat & logFormat)
+{
     defaultLogFormat = logFormat;
-    createDefaultLogger();
-}
-
-void createDefaultLogger() {
     logger = makeDefaultLogger();
 }
 

src/libmain/loggers.hh

@@ -16,6 +16,4 @@ enum class LogFormat {
 void setLogFormat(const std::string & logFormatStr);
 void setLogFormat(const LogFormat & logFormat);
 
-void createDefaultLogger();
-
 }

src/libmain/progress-bar.cc

@@ -73,8 +73,13 @@ private:
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
 
         bool active = true;
-        bool paused = false;
+        size_t suspensions = 0;
         bool haveUpdate = true;
+
+        bool isPaused() const
+        {
+            return suspensions > 0;
+        }
     };
 
     /** Helps avoid unnecessary redraws, see `redraw()` */
@@ -117,29 +122,43 @@ public:
     {
         {
             auto state(state_.lock());
-            if (!state->active) return;
-            state->active = false;
-            writeToStderr("\r\e[K");
-            updateCV.notify_one();
-            quitCV.notify_one();
+            if (state->active) {
+                state->active = false;
+                writeToStderr("\r\e[K");
+                updateCV.notify_one();
+                quitCV.notify_one();
+            }
         }
-        updateThread.join();
+        if (updateThread.joinable())
+            updateThread.join();
     }
 
     void pause() override {
         auto state (state_.lock());
-        state->paused = true;
+        state->suspensions++;
+        if (state->suspensions > 1) {
+            // already paused
+            return;
+        }
+
         if (state->active)
             writeToStderr("\r\e[K");
     }
 
     void resume() override {
         auto state (state_.lock());
-        state->paused = false;
-        if (state->active)
-            writeToStderr("\r\e[K");
-        state->haveUpdate = true;
-        updateCV.notify_one();
+        if (state->suspensions == 0) {
+            log(lvlError, "nix::ProgressBar: resume() called without a matching preceding pause(). This is a bug.");
+            return;
+        } else {
+            state->suspensions--;
+        }
+        if (state->suspensions == 0) {
+            if (state->active)
+                writeToStderr("\r\e[K");
+            state->haveUpdate = true;
+            updateCV.notify_one();
+        }
     }
 
     bool isVerbose() override
@@ -381,7 +400,7 @@ public:
         auto nextWakeup = std::chrono::milliseconds::max();
 
         state.haveUpdate = false;
-        if (state.paused || !state.active) return nextWakeup;
+        if (state.isPaused() || !state.active) return nextWakeup;
 
         std::string line;
 
@@ -553,21 +572,9 @@ public:
     }
 };
 
-Logger * makeProgressBar()
+std::unique_ptr<Logger> makeProgressBar()
 {
-    return new ProgressBar(isTTY());
-}
-
-void startProgressBar()
-{
-    logger = makeProgressBar();
-}
-
-void stopProgressBar()
-{
-    auto progressBar = dynamic_cast<ProgressBar *>(logger);
-    if (progressBar) progressBar->stop();
-
+    return std::make_unique<ProgressBar>(isTTY());
 }
 
 }

src/libmain/progress-bar.hh

@@ -5,10 +5,6 @@
 
 namespace nix {
 
-Logger * makeProgressBar();
-
-void startProgressBar();
-
-void stopProgressBar();
+std::unique_ptr<Logger> makeProgressBar();
 
 }

src/libmain/shared.cc

@@ -361,7 +361,7 @@ RunPager::RunPager()
     if (!pager) pager = getenv("PAGER");
     if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return;
 
-    stopProgressBar();
+    logger->stop();
 
     Pipe toPager;
     toPager.create();

src/libstore-tests/derivation-advanced-attrs.cc

@@ -3,13 +3,15 @@
 
 #include "experimental-features.hh"
 #include "derivations.hh"
-
-#include "tests/libstore.hh"
-#include "tests/characterization.hh"
+#include "derivations.hh"
+#include "derivation-options.hh"
 #include "parsed-derivations.hh"
 #include "types.hh"
 #include "json-utils.hh"
 
+#include "tests/libstore.hh"
+#include "tests/characterization.hh"
+
 namespace nix {
 
 using nlohmann::json;
@@ -80,21 +82,30 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults)
         auto drvPath = writeDerivation(*store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
-        EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "");
-        EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false);
-        EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings());
-        EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings());
-        EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false);
-        EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt);
-        EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt);
-        EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt);
-        EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt);
-        EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet());
-        EXPECT_EQ(parsedDrv.canBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.willBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.substitutesAllowed(), true);
-        EXPECT_EQ(parsedDrv.useUidRange(), false);
+        EXPECT_TRUE(!parsedDrv.hasStructuredAttrs());
+
+        EXPECT_EQ(options.additionalSandboxProfile, "");
+        EXPECT_EQ(options.noChroot, false);
+        EXPECT_EQ(options.impureHostDeps, StringSet{});
+        EXPECT_EQ(options.impureEnvVars, StringSet{});
+        EXPECT_EQ(options.allowLocalNetworking, false);
+        {
+            auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
+            ASSERT_TRUE(checksForAllOutputs_ != nullptr);
+            auto & checksForAllOutputs = *checksForAllOutputs_;
+
+            EXPECT_EQ(checksForAllOutputs.allowedReferences, std::nullopt);
+            EXPECT_EQ(checksForAllOutputs.allowedRequisites, std::nullopt);
+            EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{});
+            EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{});
+        }
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet());
+        EXPECT_EQ(options.canBuildLocally(*store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), true);
+        EXPECT_EQ(options.useUidRange(got), false);
     });
 };
 
@@ -106,29 +117,36 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes)
         auto drvPath = writeDerivation(*store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
         StringSet systemFeatures{"rainbow", "uid-range"};
 
-        EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle");
-        EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true);
-        EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"});
-        EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"});
-        EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true);
-        EXPECT_EQ(
-            parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
-        EXPECT_EQ(
-            parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
-        EXPECT_EQ(
-            parsedDrv.getStringsAttr("disallowedReferences"),
-            Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
-        EXPECT_EQ(
-            parsedDrv.getStringsAttr("disallowedRequisites"),
-            Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
-        EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures);
-        EXPECT_EQ(parsedDrv.canBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.willBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.substitutesAllowed(), false);
-        EXPECT_EQ(parsedDrv.useUidRange(), true);
+        EXPECT_TRUE(!parsedDrv.hasStructuredAttrs());
+
+        EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
+        EXPECT_EQ(options.noChroot, true);
+        EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"});
+        EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"});
+        EXPECT_EQ(options.allowLocalNetworking, true);
+        {
+            auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
+            ASSERT_TRUE(checksForAllOutputs_ != nullptr);
+            auto & checksForAllOutputs = *checksForAllOutputs_;
+
+            EXPECT_EQ(
+                checksForAllOutputs.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+            EXPECT_EQ(
+                checksForAllOutputs.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+            EXPECT_EQ(
+                checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+            EXPECT_EQ(
+                checksForAllOutputs.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+        }
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
+        EXPECT_EQ(options.canBuildLocally(*store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), false);
+        EXPECT_EQ(options.useUidRange(got), true);
     });
 };
 
@@ -140,27 +158,29 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr
         auto drvPath = writeDerivation(*store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
-        EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "");
-        EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false);
-        EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings());
-        EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings());
-        EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false);
+        EXPECT_TRUE(parsedDrv.hasStructuredAttrs());
+
+        EXPECT_EQ(options.additionalSandboxProfile, "");
+        EXPECT_EQ(options.noChroot, false);
+        EXPECT_EQ(options.impureHostDeps, StringSet{});
+        EXPECT_EQ(options.impureEnvVars, StringSet{});
+        EXPECT_EQ(options.allowLocalNetworking, false);
 
         {
-            auto structuredAttrs_ = parsedDrv.getStructuredAttrs();
-            ASSERT_TRUE(structuredAttrs_);
-            auto & structuredAttrs = *structuredAttrs_;
+            auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks);
+            ASSERT_TRUE(checksPerOutput_ != nullptr);
+            auto & checksPerOutput = *checksPerOutput_;
 
-            auto outputChecks_ = get(structuredAttrs, "outputChecks");
-            ASSERT_FALSE(outputChecks_);
+            EXPECT_EQ(checksPerOutput.size(), 0);
         }
 
-        EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet());
-        EXPECT_EQ(parsedDrv.canBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.willBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.substitutesAllowed(), true);
-        EXPECT_EQ(parsedDrv.useUidRange(), false);
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet());
+        EXPECT_EQ(options.canBuildLocally(*store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), true);
+        EXPECT_EQ(options.useUidRange(got), false);
     });
 };
 
@@ -172,62 +192,52 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr
         auto drvPath = writeDerivation(*store, got, NoRepair, true);
 
         ParsedDerivation parsedDrv(drvPath, got);
+        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
 
         StringSet systemFeatures{"rainbow", "uid-range"};
 
-        EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle");
-        EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true);
-        EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"});
-        EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"});
-        EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true);
+        EXPECT_TRUE(parsedDrv.hasStructuredAttrs());
+
+        EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
+        EXPECT_EQ(options.noChroot, true);
+        EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"});
+        EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"});
+        EXPECT_EQ(options.allowLocalNetworking, true);
 
         {
-            auto structuredAttrs_ = parsedDrv.getStructuredAttrs();
-            ASSERT_TRUE(structuredAttrs_);
-            auto & structuredAttrs = *structuredAttrs_;
-
-            auto outputChecks_ = get(structuredAttrs, "outputChecks");
-            ASSERT_TRUE(outputChecks_);
-            auto & outputChecks = *outputChecks_;
-
             {
-                auto output_ = get(outputChecks, "out");
+                auto output_ = get(std::get<1>(options.outputChecks), "out");
                 ASSERT_TRUE(output_);
                 auto & output = *output_;
-                EXPECT_EQ(
-                    get(output, "allowedReferences")->get<Strings>(),
-                    Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
-                EXPECT_EQ(
-                    get(output, "allowedRequisites")->get<Strings>(),
-                    Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+
+                EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
+                EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"});
             }
 
             {
-                auto output_ = get(outputChecks, "bin");
+                auto output_ = get(std::get<1>(options.outputChecks), "bin");
                 ASSERT_TRUE(output_);
                 auto & output = *output_;
-                EXPECT_EQ(
-                    get(output, "disallowedReferences")->get<Strings>(),
-                    Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
-                EXPECT_EQ(
-                    get(output, "disallowedRequisites")->get<Strings>(),
-                    Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+
+                EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
+                EXPECT_EQ(output.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"});
             }
 
             {
-                auto output_ = get(outputChecks, "dev");
+                auto output_ = get(std::get<1>(options.outputChecks), "dev");
                 ASSERT_TRUE(output_);
                 auto & output = *output_;
-                EXPECT_EQ(get(output, "maxSize")->get<uint64_t>(), 789);
-                EXPECT_EQ(get(output, "maxClosureSize")->get<uint64_t>(), 5909);
+
+                EXPECT_EQ(output.maxSize, 789);
+                EXPECT_EQ(output.maxClosureSize, 5909);
             }
         }
 
-        EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures);
-        EXPECT_EQ(parsedDrv.canBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.willBuildLocally(*store), false);
-        EXPECT_EQ(parsedDrv.substitutesAllowed(), false);
-        EXPECT_EQ(parsedDrv.useUidRange(), true);
+        EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures);
+        EXPECT_EQ(options.canBuildLocally(*store, got), false);
+        EXPECT_EQ(options.willBuildLocally(*store, got), false);
+        EXPECT_EQ(options.substitutesAllowed(), false);
+        EXPECT_EQ(options.useUidRange(got), true);
     });
 };
 

src/libstore/build/derivation-goal.cc

@@ -36,14 +36,6 @@
 
 namespace nix {
 
-Goal::Co DerivationGoal::init() {
-    if (useDerivation) {
-        co_return getDerivation();
-    } else {
-        co_return haveDerivation();
-    }
-}
-
 DerivationGoal::DerivationGoal(const StorePath & drvPath,
     const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
     : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
@@ -141,50 +133,44 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
-Goal::Co DerivationGoal::getDerivation()
-{
+Goal::Co DerivationGoal::init() {
     trace("init");
 
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
-        co_return loadDerivation();
-    }
+    if (useDerivation) {
+        /* The first thing to do is to make sure that the derivation
+           exists.  If it doesn't, it may be created through a
+           substitute. */
 
-    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
-
-    co_await Suspend{};
-    co_return loadDerivation();
-}
-
-
-Goal::Co DerivationGoal::loadDerivation()
-{
-    trace("loading derivation");
-
-    if (nrFailed != 0) {
-        co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
-    }
-
-    /* `drvPath' should already be a root, but let's be on the safe
-       side: if the user forgot to make it a root, we wouldn't want
-       things being garbage collected while we're busy. */
-    worker.evalStore.addTempRoot(drvPath);
-
-    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
-
-         - Resolved derivation are resolved against main store realisations, and so must be stored there.
-
-         - Dynamic derivations are built, and so are found in the main store.
-     */
-    for (auto * drvStore : { &worker.evalStore, &worker.store }) {
-        if (drvStore->isValidPath(drvPath)) {
-            drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
-            break;
+        if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) {
+            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
+            co_await Suspend{};
         }
+
+        trace("loading derivation");
+
+        if (nrFailed != 0) {
+            co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
+        }
+
+        /* `drvPath' should already be a root, but let's be on the safe
+           side: if the user forgot to make it a root, we wouldn't want
+           things being garbage collected while we're busy. */
+        worker.evalStore.addTempRoot(drvPath);
+
+        /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
+
+             - Resolved derivation are resolved against main store realisations, and so must be stored there.
+
+             - Dynamic derivations are built, and so are found in the main store.
+         */
+        for (auto * drvStore : { &worker.evalStore, &worker.store }) {
+            if (drvStore->isValidPath(drvPath)) {
+                drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+                break;
+            }
+        }
+        assert(drv);
     }
-    assert(drv);
 
     co_return haveDerivation();
 }
@@ -195,58 +181,64 @@ Goal::Co DerivationGoal::haveDerivation()
     trace("have derivation");
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+    drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(*parsedDrv));
 
     if (!drv->type().hasKnownOutputPaths())
         experimentalFeatureSettings.require(Xp::CaDerivations);
 
-    if (drv->type().isImpure()) {
-        experimentalFeatureSettings.require(Xp::ImpureDerivations);
-
-        for (auto & [outputName, output] : drv->outputs) {
-            auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
-            assert(!worker.store.isValidPath(randomPath));
-            initialOutputs.insert({
-                outputName,
-                InitialOutput {
-                    .wanted = true,
-                    .outputHash = impureOutputHash,
-                    .known = InitialOutputStatus {
-                        .path = randomPath,
-                        .status = PathStatus::Absent
-                    }
-                }
-            });
-        }
-
-        co_return gaveUpOnSubstitution();
-    }
-
     for (auto & i : drv->outputsAndOptPaths(worker.store))
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
 
-    auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
-    for (auto & [outputName, outputHash] : outputHashes)
-        initialOutputs.insert({
-            outputName,
-            InitialOutput {
+    {
+        bool impure = drv->type().isImpure();
+
+        if (impure) experimentalFeatureSettings.require(Xp::ImpureDerivations);
+
+        auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
+        for (auto & [outputName, outputHash] : outputHashes) {
+            InitialOutput v{
                 .wanted = true, // Will be refined later
                 .outputHash = outputHash
+            };
+
+            /* TODO we might want to also allow randomizing the paths
+               for regular CA derivations, e.g. for sake of checking
+               determinism. */
+            if (impure) {
+                v.known = InitialOutputStatus {
+                    .path = StorePath::random(outputPathName(drv->name, outputName)),
+                    .status = PathStatus::Absent,
+                };
             }
-        });
 
-    /* Check what outputs paths are not already valid. */
-    auto [allValid, validOutputs] = checkPathValidity();
+            initialOutputs.insert({
+                outputName,
+                std::move(v),
+            });
+        }
 
-    /* If they are all valid, then we're done. */
-    if (allValid && buildMode == bmNormal) {
-        co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
+        if (impure) {
+            /* We don't yet have any safe way to cache an impure derivation at
+               this step. */
+            co_return gaveUpOnSubstitution();
+        }
+    }
+
+    {
+        /* Check what outputs paths are not already valid. */
+        auto [allValid, validOutputs] = checkPathValidity();
+
+        /* If they are all valid, then we're done. */
+        if (allValid && buildMode == bmNormal) {
+            co_return done(BuildResult::AlreadyValid, std::move(validOutputs));
+        }
     }
 
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
+    if (settings.useSubstitutes && drvOptions->substitutesAllowed())
         for (auto & [outputName, status] : initialOutputs) {
             if (!status.wanted) continue;
             if (!status.known)
@@ -268,12 +260,7 @@ Goal::Co DerivationGoal::haveDerivation()
         }
 
     if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
-    co_return outputsSubstitutionTried();
-}
 
-
-Goal::Co DerivationGoal::outputsSubstitutionTried()
-{
     trace("all outputs substituted (maybe)");
 
     assert(!drv->type().isImpure());
@@ -399,84 +386,7 @@ Goal::Co DerivationGoal::gaveUpOnSubstitution()
     }
 
     if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */
-    co_return inputsRealised();
-}
 
-
-Goal::Co DerivationGoal::repairClosure()
-{
-    assert(!drv->type().isImpure());
-
-    /* If we're repairing, we now know that our own outputs are valid.
-       Now check whether the other paths in the outputs closure are
-       good.  If not, then start derivation goals for the derivations
-       that produced those outputs. */
-
-    /* Get the output closure. */
-    auto outputs = queryDerivationOutputMap();
-    StorePathSet outputClosure;
-    for (auto & i : outputs) {
-        if (!wantedOutputs.contains(i.first)) continue;
-        worker.store.computeFSClosure(i.second, outputClosure);
-    }
-
-    /* Filter out our own outputs (which we have already checked). */
-    for (auto & i : outputs)
-        outputClosure.erase(i.second);
-
-    /* Get all dependencies of this derivation so that we know which
-       derivation is responsible for which path in the output
-       closure. */
-    StorePathSet inputClosure;
-    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
-    std::map<StorePath, StorePath> outputsToDrv;
-    for (auto & i : inputClosure)
-        if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
-            for (auto & j : depOutputs)
-                if (j.second)
-                    outputsToDrv.insert_or_assign(*j.second, i);
-        }
-
-    /* Check each path (slow!). */
-    for (auto & i : outputClosure) {
-        if (worker.pathContentsGood(i)) continue;
-        printError(
-            "found corrupted or missing path '%s' in the output closure of '%s'",
-            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        auto drvPath2 = outputsToDrv.find(i);
-        if (drvPath2 == outputsToDrv.end())
-            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
-        else
-            addWaitee(worker.makeGoal(
-                DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(drvPath2->second),
-                    .outputs = OutputsSpec::All { },
-                },
-                bmRepair));
-    }
-
-    if (waitees.empty()) {
-        co_return done(BuildResult::AlreadyValid, assertPathValidity());
-    } else {
-        co_await Suspend{};
-        co_return closureRepaired();
-    }
-}
-
-
-Goal::Co DerivationGoal::closureRepaired()
-{
-    trace("closure repaired");
-    if (nrFailed > 0)
-        throw Error("some paths in the output closure of derivation '%s' could not be repaired",
-            worker.store.printStorePath(drvPath));
-    co_return done(BuildResult::AlreadyValid, assertPathValidity());
-}
-
-
-Goal::Co DerivationGoal::inputsRealised()
-{
     trace("all inputs realised");
 
     if (nrFailed != 0) {
@@ -718,7 +628,7 @@ Goal::Co DerivationGoal::tryToBuild()
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
     bool buildLocally =
-        (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
+        (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
         && settings.maxBuildJobs.get() != 0;
 
     if (!buildLocally) {
@@ -766,6 +676,73 @@ Goal::Co DerivationGoal::tryLocalBuild() {
 }
 
 
+Goal::Co DerivationGoal::repairClosure()
+{
+    assert(!drv->type().isImpure());
+
+    /* If we're repairing, we now know that our own outputs are valid.
+       Now check whether the other paths in the outputs closure are
+       good.  If not, then start derivation goals for the derivations
+       that produced those outputs. */
+
+    /* Get the output closure. */
+    auto outputs = queryDerivationOutputMap();
+    StorePathSet outputClosure;
+    for (auto & i : outputs) {
+        if (!wantedOutputs.contains(i.first)) continue;
+        worker.store.computeFSClosure(i.second, outputClosure);
+    }
+
+    /* Filter out our own outputs (which we have already checked). */
+    for (auto & i : outputs)
+        outputClosure.erase(i.second);
+
+    /* Get all dependencies of this derivation so that we know which
+       derivation is responsible for which path in the output
+       closure. */
+    StorePathSet inputClosure;
+    if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure);
+    std::map<StorePath, StorePath> outputsToDrv;
+    for (auto & i : inputClosure)
+        if (i.isDerivation()) {
+            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
+            for (auto & j : depOutputs)
+                if (j.second)
+                    outputsToDrv.insert_or_assign(*j.second, i);
+        }
+
+    /* Check each path (slow!). */
+    for (auto & i : outputClosure) {
+        if (worker.pathContentsGood(i)) continue;
+        printError(
+            "found corrupted or missing path '%s' in the output closure of '%s'",
+            worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
+        auto drvPath2 = outputsToDrv.find(i);
+        if (drvPath2 == outputsToDrv.end())
+            addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
+        else
+            addWaitee(worker.makeGoal(
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(drvPath2->second),
+                    .outputs = OutputsSpec::All { },
+                },
+                bmRepair));
+    }
+
+    if (waitees.empty()) {
+        co_return done(BuildResult::AlreadyValid, assertPathValidity());
+    } else {
+        co_await Suspend{};
+
+        trace("closure repaired");
+        if (nrFailed > 0)
+            throw Error("some paths in the output closure of derivation '%s' could not be repaired",
+                worker.store.printStorePath(drvPath));
+        co_return done(BuildResult::AlreadyValid, assertPathValidity());
+    }
+}
+
+
 static void chmod_(const Path & path, mode_t mode)
 {
     if (chmod(path.c_str(), mode) == -1)
@@ -1145,7 +1122,7 @@ HookReply DerivationGoal::tryBuildHook()
             << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0)
             << drv->platform
             << worker.store.printStorePath(drvPath)
-            << parsedDrv->getRequiredSystemFeatures();
+            << drvOptions->getRequiredSystemFeatures(*drv);
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating
@@ -1247,7 +1224,7 @@ SingleDrvOutputs DerivationGoal::registerOutputs()
        to do anything here.
 
        We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
+       floating content-addressing derivations this isn't the case.
      */
     return assertPathValidity();
 }

src/libstore/build/derivation-goal.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "parsed-derivations.hh"
+#include "derivation-options.hh"
 #ifndef _WIN32
 #  include "user-lock.hh"
 #endif
@@ -80,7 +81,7 @@ struct DerivationGoal : public Goal
     /**
      * Mapping from input derivations + output names to actual store
      * paths. This is filled in by waiteeDone() as each dependency
-     * finishes, before inputsRealised() is reached.
+     * finishes, before `trace("all inputs realised")` is reached.
      */
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
@@ -143,6 +144,7 @@ struct DerivationGoal : public Goal
     std::unique_ptr<Derivation> drv;
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
+    std::unique_ptr<DerivationOptions> drvOptions;
 
     /**
      * The remainder is state held during the build.
@@ -233,13 +235,8 @@ struct DerivationGoal : public Goal
      * The states.
      */
     Co init() override;
-    Co getDerivation();
-    Co loadDerivation();
     Co haveDerivation();
-    Co outputsSubstitutionTried();
     Co gaveUpOnSubstitution();
-    Co closureRepaired();
-    Co inputsRealised();
     Co tryToBuild();
     virtual Co tryLocalBuild();
     Co buildDone();

src/libstore/ca-specific-schema.sql

@@ -1,4 +1,4 @@
--- Extension of the sql schema for content-addressed derivations.
+-- Extension of the sql schema for content-addressing derivations.
 -- Won't be loaded unless the experimental feature `ca-derivations`
 -- is enabled
 

src/libstore/daemon.cc

@@ -593,7 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         auto drvType = drv.type();
 
-        /* Content-addressed derivations are trustless because their output paths
+        /* Content-addressing derivations are trustless because their output paths
            are verified by their content alone, so any derivation is free to
            try to produce such a path.
 
@@ -1041,11 +1041,15 @@ void processConnection(
     conn.protoVersion = protoVersion;
     conn.features = features;
 
-    auto tunnelLogger = new TunnelLogger(conn.to, protoVersion);
-    auto prevLogger = nix::logger;
+    auto tunnelLogger_ = std::make_unique<TunnelLogger>(conn.to, protoVersion);
+    auto tunnelLogger = tunnelLogger_.get();
+    std::unique_ptr<Logger> prevLogger_;
+    auto prevLogger = logger.get();
     // FIXME
-    if (!recursive)
-        logger = tunnelLogger;
+    if (!recursive) {
+        prevLogger_ = std::move(logger);
+        logger = std::move(tunnelLogger_);
+    }
 
     unsigned int opCount = 0;
 

src/libstore/derivation-options.cc

@@ -0,0 +1,274 @@
+#include "derivation-options.hh"
+#include "json-utils.hh"
+#include "parsed-derivations.hh"
+#include "types.hh"
+#include "util.hh"
+#include <optional>
+#include <string>
+#include <variant>
+
+namespace nix {
+
+using OutputChecks = DerivationOptions::OutputChecks;
+
+using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
+
+DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn)
+{
+    DerivationOptions defaults = {};
+
+    auto structuredAttrs = parsed.structuredAttrs.get();
+
+    if (shouldWarn && structuredAttrs) {
+        if (get(*structuredAttrs, "allowedReferences")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
+        }
+        if (get(*structuredAttrs, "allowedRequisites")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
+        }
+        if (get(*structuredAttrs, "disallowedRequisites")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
+        }
+        if (get(*structuredAttrs, "disallowedReferences")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
+        }
+        if (get(*structuredAttrs, "maxSize")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
+        }
+        if (get(*structuredAttrs, "maxClosureSize")) {
+            warn(
+                "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
+        }
+    }
+
+    return {
+        .outputChecks = [&]() -> OutputChecksVariant {
+            if (auto structuredAttrs = parsed.structuredAttrs.get()) {
+                std::map<std::string, OutputChecks> res;
+                if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
+                    for (auto & [outputName, output] : getObject(*outputChecks)) {
+                        OutputChecks checks;
+
+                        if (auto maxSize = get(output, "maxSize"))
+                            checks.maxSize = maxSize->get<uint64_t>();
+
+                        if (auto maxClosureSize = get(output, "maxClosureSize"))
+                            checks.maxClosureSize = maxClosureSize->get<uint64_t>();
+
+                        auto get_ = [&](const std::string & name) -> std::optional<StringSet> {
+                            if (auto i = get(output, name)) {
+                                StringSet res;
+                                for (auto j = i->begin(); j != i->end(); ++j) {
+                                    if (!j->is_string())
+                                        throw Error("attribute '%s' must be a list of strings", name);
+                                    res.insert(j->get<std::string>());
+                                }
+                                checks.disallowedRequisites = res;
+                                return res;
+                            }
+                            return {};
+                        };
+
+                        checks.allowedReferences = get_("allowedReferences");
+                        checks.allowedRequisites = get_("allowedRequisites");
+                        checks.disallowedReferences = get_("disallowedReferences").value_or(StringSet{});
+                        checks.disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{});
+                        ;
+
+                        res.insert_or_assign(outputName, std::move(checks));
+                    }
+                }
+                return res;
+            } else {
+                return OutputChecks{
+                    // legacy non-structured-attributes case
+                    .ignoreSelfRefs = true,
+                    .allowedReferences = parsed.getStringSetAttr("allowedReferences"),
+                    .disallowedReferences = parsed.getStringSetAttr("disallowedReferences").value_or(StringSet{}),
+                    .allowedRequisites = parsed.getStringSetAttr("allowedRequisites"),
+                    .disallowedRequisites = parsed.getStringSetAttr("disallowedRequisites").value_or(StringSet{}),
+                };
+            }
+        }(),
+        .unsafeDiscardReferences =
+            [&] {
+                std::map<std::string, bool> res;
+
+                if (auto structuredAttrs = parsed.structuredAttrs.get()) {
+                    if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
+                        for (auto & [outputName, output] : getObject(*udr)) {
+                            if (!output.is_boolean())
+                                throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
+                            res.insert_or_assign(outputName, output.get<bool>());
+                        }
+                    }
+                }
+
+                return res;
+            }(),
+        .passAsFile =
+            [&] {
+                StringSet res;
+                if (auto * passAsFileString = get(parsed.drv.env, "passAsFile")) {
+                    if (parsed.hasStructuredAttrs()) {
+                        if (shouldWarn) {
+                            warn(
+                                "'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file");
+                        }
+                    } else {
+                        res = tokenizeString<StringSet>(*passAsFileString);
+                    }
+                }
+                return res;
+            }(),
+        .additionalSandboxProfile =
+            parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile),
+        .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot),
+        .impureHostDeps = parsed.getStringSetAttr("__impureHostDeps").value_or(defaults.impureHostDeps),
+        .impureEnvVars = parsed.getStringSetAttr("impureEnvVars").value_or(defaults.impureEnvVars),
+        .allowLocalNetworking = parsed.getBoolAttr("__darwinAllowLocalNetworking", defaults.allowLocalNetworking),
+        .requiredSystemFeatures =
+            parsed.getStringSetAttr("requiredSystemFeatures").value_or(defaults.requiredSystemFeatures),
+        .preferLocalBuild = parsed.getBoolAttr("preferLocalBuild", defaults.preferLocalBuild),
+        .allowSubstitutes = parsed.getBoolAttr("allowSubstitutes", defaults.allowSubstitutes),
+    };
+}
+
+StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const
+{
+    // FIXME: cache this?
+    StringSet res;
+    for (auto & i : requiredSystemFeatures)
+        res.insert(i);
+    if (!drv.type().hasKnownOutputPaths())
+        res.insert("ca-derivations");
+    return res;
+}
+
+bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
+{
+    if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
+        && !drv.isBuiltin())
+        return false;
+
+    if (settings.maxBuildJobs.get() == 0 && !drv.isBuiltin())
+        return false;
+
+    for (auto & feature : getRequiredSystemFeatures(drv))
+        if (!localStore.systemFeatures.get().count(feature))
+            return false;
+
+    return true;
+}
+
+bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
+{
+    return preferLocalBuild && canBuildLocally(localStore, drv);
+}
+
+bool DerivationOptions::substitutesAllowed() const
+{
+    return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
+}
+
+bool DerivationOptions::useUidRange(const BasicDerivation & drv) const
+{
+    return getRequiredSystemFeatures(drv).count("uid-range");
+}
+
+}
+
+namespace nlohmann {
+
+using namespace nix;
+
+DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json)
+{
+    return {
+        .outputChecks = [&]() -> OutputChecksVariant {
+            auto outputChecks = getObject(valueAt(json, "outputChecks"));
+
+            auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
+            auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
+
+            if (forAllOutputsOpt && !perOutputOpt) {
+                return static_cast<OutputChecks>(*forAllOutputsOpt);
+            } else if (perOutputOpt && !forAllOutputsOpt) {
+                return static_cast<std::map<std::string, OutputChecks>>(*perOutputOpt);
+            } else {
+                throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required");
+            }
+        }(),
+
+        .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"),
+        .passAsFile = getStringSet(valueAt(json, "passAsFile")),
+
+        .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")),
+        .noChroot = getBoolean(valueAt(json, "noChroot")),
+        .impureHostDeps = getStringSet(valueAt(json, "impureHostDeps")),
+        .impureEnvVars = getStringSet(valueAt(json, "impureEnvVars")),
+        .allowLocalNetworking = getBoolean(valueAt(json, "allowLocalNetworking")),
+
+        .requiredSystemFeatures = getStringSet(valueAt(json, "requiredSystemFeatures")),
+        .preferLocalBuild = getBoolean(valueAt(json, "preferLocalBuild")),
+        .allowSubstitutes = getBoolean(valueAt(json, "allowSubstitutes")),
+    };
+}
+
+void adl_serializer<DerivationOptions>::to_json(json & json, DerivationOptions o)
+{
+    json["outputChecks"] = std::visit(
+        overloaded{
+            [&](const OutputChecks & checks) {
+                nlohmann::json outputChecks;
+                outputChecks["forAllOutputs"] = checks;
+                return outputChecks;
+            },
+            [&](const std::map<std::string, OutputChecks> & checksPerOutput) {
+                nlohmann::json outputChecks;
+                outputChecks["perOutput"] = checksPerOutput;
+                return outputChecks;
+            },
+        },
+        o.outputChecks);
+
+    json["unsafeDiscardReferences"] = o.unsafeDiscardReferences;
+    json["passAsFile"] = o.passAsFile;
+
+    json["additionalSandboxProfile"] = o.additionalSandboxProfile;
+    json["noChroot"] = o.noChroot;
+    json["impureHostDeps"] = o.impureHostDeps;
+    json["impureEnvVars"] = o.impureEnvVars;
+    json["allowLocalNetworking"] = o.allowLocalNetworking;
+
+    json["requiredSystemFeatures"] = o.requiredSystemFeatures;
+    json["preferLocalBuild"] = o.preferLocalBuild;
+    json["allowSubstitutes"] = o.allowSubstitutes;
+}
+
+DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json)
+{
+    return {
+        .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
+        .allowedReferences = nullableValueAt(json, "allowedReferences"),
+        .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
+        .allowedRequisites = nullableValueAt(json, "allowedRequisites"),
+        .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
+    };
+}
+
+void adl_serializer<DerivationOptions::OutputChecks>::to_json(json & json, DerivationOptions::OutputChecks c)
+{
+    json["ignoreSelfRefs"] = c.ignoreSelfRefs;
+    json["allowedReferences"] = c.allowedReferences;
+    json["disallowedReferences"] = c.disallowedReferences;
+    json["allowedRequisites"] = c.allowedRequisites;
+    json["disallowedRequisites"] = c.disallowedRequisites;
+}
+
+}

src/libstore/derivation-options.hh

@@ -0,0 +1,185 @@
+#pragma once
+///@file
+
+#include <cstdint>
+#include <nlohmann/json.hpp>
+#include <optional>
+#include <variant>
+
+#include "types.hh"
+#include "json-impls.hh"
+
+namespace nix {
+
+class Store;
+struct BasicDerivation;
+class ParsedDerivation;
+
+/**
+ * This represents all the special options on a `Derivation`.
+ *
+ * Currently, these options are parsed from the environment variables
+ * with the aid of `ParsedDerivation`.
+ *
+ * The first goal of this data type is to make sure that no other code
+ * uses `ParsedDerivation` to ad-hoc parse some additional options. That
+ * ensures this data type is up to date and fully correct.
+ *
+ * The second goal of this data type is to allow an alternative to
+ * hackily parsing the options from the environment variables. The ATerm
+ * format cannot change, but in alternatives to it (like the JSON
+ * format), we have the option of instead storing the options
+ * separately. That would be nice to separate concerns, and not make any
+ * environment variable names magical.
+ */
+struct DerivationOptions
+{
+    struct OutputChecks
+    {
+        bool ignoreSelfRefs = false;
+        std::optional<uint64_t> maxSize, maxClosureSize;
+
+        /**
+         * env: allowedReferences
+         *
+         * A value of `nullopt` indicates that the check is skipped.
+         * This means that all references are allowed.
+         */
+        std::optional<StringSet> allowedReferences;
+
+        /**
+         * env: disallowedReferences
+         *
+         * No needed for `std::optional`, because skipping the check is
+         * the same as disallowing the references.
+         */
+        StringSet disallowedReferences;
+
+        /**
+         * env: allowedRequisites
+         *
+         * See `allowedReferences`
+         */
+        std::optional<StringSet> allowedRequisites;
+
+        /**
+         * env: disallowedRequisites
+         *
+         * See `disallowedReferences`
+         */
+        StringSet disallowedRequisites;
+
+        bool operator==(const OutputChecks &) const = default;
+    };
+
+    /**
+     * Either one set of checks for all outputs, or separate checks
+     * per-output.
+     */
+    std::variant<OutputChecks, std::map<std::string, OutputChecks>> outputChecks = OutputChecks{};
+
+    /**
+     * Whether to avoid scanning for references for a given output.
+     */
+    std::map<std::string, bool> unsafeDiscardReferences;
+
+    /**
+     * In non-structured mode, all bindings specified in the derivation
+     * go directly via the environment, except those listed in the
+     * passAsFile attribute. Those are instead passed as file names
+     * pointing to temporary files containing the contents.
+     *
+     * Note that passAsFile is ignored in structure mode because it's
+     * not needed (attributes are not passed through the environment, so
+     * there is no size constraint).
+     */
+    StringSet passAsFile;
+
+    /**
+     * env: __sandboxProfile
+     *
+     * Just for Darwin
+     */
+    std::string additionalSandboxProfile = "";
+
+    /**
+     * env: __noChroot
+     *
+     * Derivation would like to opt out of the sandbox.
+     *
+     * Builder is free to not respect this wish (because it is
+     * insecure) and fail the build instead.
+     */
+    bool noChroot = false;
+
+    /**
+     * env: __impureHostDeps
+     */
+    StringSet impureHostDeps = {};
+
+    /**
+     * env: impureEnvVars
+     */
+    StringSet impureEnvVars = {};
+
+    /**
+     * env: __darwinAllowLocalNetworking
+     *
+     * Just for Darwin
+     */
+    bool allowLocalNetworking = false;
+
+    /**
+     * env: requiredSystemFeatures
+     */
+    StringSet requiredSystemFeatures = {};
+
+    /**
+     * env: preferLocalBuild
+     */
+    bool preferLocalBuild = false;
+
+    /**
+     * env: allowSubstitutes
+     */
+    bool allowSubstitutes = true;
+
+    bool operator==(const DerivationOptions &) const = default;
+
+    /**
+     * Parse this information from its legacy encoding as part of the
+     * environment. This should not be used with nice greenfield formats
+     * (e.g. JSON) but is necessary for supporing old formats (e.g.
+     * ATerm).
+     */
+    static DerivationOptions fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn = true);
+
+    /**
+     * @param drv Must be the same derivation we parsed this from. In
+     * the future we'll flip things around so a `BasicDerivation` has
+     * `DerivationOptions` instead.
+     */
+    StringSet getRequiredSystemFeatures(const BasicDerivation & drv) const;
+
+    /**
+     * @param drv See note on `getRequiredSystemFeatures`
+     */
+    bool canBuildLocally(Store & localStore, const BasicDerivation & drv) const;
+
+    /**
+     * @param drv See note on `getRequiredSystemFeatures`
+     */
+    bool willBuildLocally(Store & localStore, const BasicDerivation & drv) const;
+
+    bool substitutesAllowed() const;
+
+    /**
+     * @param drv See note on `getRequiredSystemFeatures`
+     */
+    bool useUidRange(const BasicDerivation & drv) const;
+};
+
+};
+
+JSON_IMPL(DerivationOptions);
+JSON_IMPL(DerivationOptions::OutputChecks)

src/libstore/derivations.cc

@@ -300,7 +300,7 @@ static DerivationOutput parseDerivationOutput(
         } else {
             xpSettings.require(Xp::CaDerivations);
             if (pathS != "")
-                throw FormatError("content-addressed derivation output should not specify output path");
+                throw FormatError("content-addressing derivation output should not specify output path");
             return DerivationOutput::CAFloating {
                 .method = std::move(method),
                 .hashAlgo = std::move(hashAlgo),
@@ -843,16 +843,6 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         };
     }
 
-    if (type.isImpure()) {
-        std::map<std::string, Hash> outputHashes;
-        for (const auto & [outputName, _] : drv.outputs)
-            outputHashes.insert_or_assign(outputName, impureOutputHash);
-        return DrvHash {
-            .hashes = outputHashes,
-            .kind = DrvHash::Kind::Deferred,
-        };
-    }
-
     auto kind = std::visit(overloaded {
         [](const DerivationType::InputAddressed & ia) {
             /* This might be a "pesimistically" deferred output, so we don't
@@ -865,7 +855,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
                 : DrvHash::Kind::Deferred;
         },
         [](const DerivationType::Impure &) -> DrvHash::Kind {
-            assert(false);
+            return DrvHash::Kind::Deferred;
         }
     }, drv.type().raw);
 

src/libstore/derivations.hh

@@ -187,7 +187,7 @@ struct DerivationType {
     };
 
     /**
-     * Content-addressed derivation types
+     * Content-addressing derivation types
      */
     struct ContentAddressed {
         /**
@@ -526,6 +526,4 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
  */
 std::string hashPlaceholder(const OutputNameView outputName);
 
-extern const Hash impureOutputHash;
-
 }

src/libstore/filetransfer.cc

@@ -94,7 +94,7 @@ struct curlFileTransfer : public FileTransfer
             : fileTransfer(fileTransfer)
             , request(request)
             , act(*logger, lvlTalkative, actFileTransfer,
-                fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
+                request.post ? "" : fmt(request.data ?  "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
             , finalSink([this](std::string_view data) {
@@ -261,7 +261,7 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
-        int progressCallback(double dltotal, double dlnow)
+        int progressCallback(curl_off_t dltotal, curl_off_t dlnow)
         {
             try {
                 act.progress(dlnow, dltotal);
@@ -271,11 +271,21 @@ struct curlFileTransfer : public FileTransfer
             return getInterrupted();
         }
 
-        static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
+        int silentProgressCallback(curl_off_t dltotal, curl_off_t dlnow)
+        {
+            return getInterrupted();
+        }
+
+        static int progressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
         {
             return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
         }
 
+        static int silentProgressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
+        {
+            return ((TransferItem *) userp)->silentProgressCallback(dltotal, dlnow);
+        }
+
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
         {
             if (type == CURLINFO_TEXT)
@@ -342,8 +352,11 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
-            curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
-            curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this);
+            if (request.post)
+                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, silentProgressCallbackWrapper);
+            else
+                curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
             curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
 
             curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
@@ -355,7 +368,10 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
             if (request.data) {
-                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                if (request.post)
+                    curl_easy_setopt(req, CURLOPT_POST, 1L);
+                else
+                    curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
                 curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
                 curl_easy_setopt(req, CURLOPT_READDATA, this);
                 curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
@@ -432,7 +448,8 @@ struct curlFileTransfer : public FileTransfer
                 if (httpStatus == 304 && result.etag == "")
                     result.etag = request.expectedETag;
 
-                act.progress(result.bodySize, result.bodySize);
+                if (!request.post)
+                    act.progress(result.bodySize, result.bodySize);
                 done = true;
                 callback(std::move(result));
             }

src/libstore/filetransfer.hh

@@ -65,6 +65,7 @@ struct FileTransferRequest
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
+    bool post = false;
     size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;

src/libstore/globals.hh

@@ -184,7 +184,7 @@ public:
         this, SYSTEM, "system",
         R"(
           The system type of the current Nix installation.
-          Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
+          Nix will only build a given [store derivation](@docroot@/glossary.md#gloss-store-derivation) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
 
           The default value is set when Nix itself is compiled for the system it will run on.
           The following system types are widely used, as Nix is actively supported on these platforms:
@@ -820,7 +820,7 @@ public:
         R"(
           System types of executables that can be run on this machine.
 
-          Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system).
+          Nix will only build a given [store derivation](@docroot@/glossary.md#gloss-store-derivation) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system).
 
           Setting this can be useful to build derivations locally on compatible machines:
           - `i686-linux` executables can be run on `x86_64-linux` machines (set by default)
@@ -1059,7 +1059,10 @@ public:
 
           1. `NIX_SSL_CERT_FILE`
           2. `SSL_CERT_FILE`
-        )"};
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 
 #if __linux__
     Setting<bool> filterSyscalls{

src/libstore/local-store.cc

@@ -172,16 +172,15 @@ LocalStore::LocalStore(
 
     /* Ensure that the store and its parents are not symlinks. */
     if (!settings.allowSymlinkedStore) {
-        Path path = realStoreDir;
-        struct stat st;
-        while (path != "/") {
-            st = lstat(path);
-            if (S_ISLNK(st.st_mode))
+        std::filesystem::path path = realStoreDir.get();
+        std::filesystem::path root = path.root_path();
+        while (path != root) {
+            if (std::filesystem::is_symlink(path))
                 throw Error(
                         "the path '%1%' is a symlink; "
                         "this is not allowed for the Nix store and its parent directories",
                         path);
-            path = dirOf(path);
+            path = path.parent_path();
         }
     }
 

src/libstore/meson.build

@@ -200,6 +200,7 @@ sources = files(
   'content-address.cc',
   'daemon.cc',
   'derivations.cc',
+  'derivation-options.cc',
   'derived-path-map.cc',
   'derived-path.cc',
   'downstream-placeholder.cc',
@@ -271,6 +272,7 @@ headers = [config_h] + files(
   'content-address.hh',
   'daemon.hh',
   'derivations.hh',
+  'derivation-options.hh',
   'derived-path-map.hh',
   'derived-path.hh',
   'downstream-placeholder.hh',

src/libstore/misc.cc

@@ -2,6 +2,7 @@
 
 #include "derivations.hh"
 #include "parsed-derivations.hh"
+#include "derivation-options.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
@@ -222,8 +223,9 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
             auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
             ParsedDerivation parsedDrv(StorePath(drvPath), *drv);
+            DerivationOptions drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv);
 
-            if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
+            if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
                 experimentalFeatureSettings.require(Xp::CaDerivations);
 
                 // If there are unknown output paths, attempt to find if the
@@ -253,7 +255,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
                 }
             }
 
-            if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
+            if (knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) {
                 auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
                 for (auto & output : invalid)
                     pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState));

src/libstore/parsed-derivations.cc

@@ -87,47 +87,12 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
     }
 }
 
-StringSet ParsedDerivation::getRequiredSystemFeatures() const
+std::optional<StringSet> ParsedDerivation::getStringSetAttr(const std::string & name) const
 {
-    // FIXME: cache this?
-    StringSet res;
-    for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
-        res.insert(i);
-    if (!drv.type().hasKnownOutputPaths())
-        res.insert("ca-derivations");
-    return res;
-}
-
-bool ParsedDerivation::canBuildLocally(Store & localStore) const
-{
-    if (drv.platform != settings.thisSystem.get()
-        && !settings.extraPlatforms.get().count(drv.platform)
-        && !drv.isBuiltin())
-        return false;
-
-    if (settings.maxBuildJobs.get() == 0
-        && !drv.isBuiltin())
-        return false;
-
-    for (auto & feature : getRequiredSystemFeatures())
-        if (!localStore.systemFeatures.get().count(feature)) return false;
-
-    return true;
-}
-
-bool ParsedDerivation::willBuildLocally(Store & localStore) const
-{
-    return getBoolAttr("preferLocalBuild") && canBuildLocally(localStore);
-}
-
-bool ParsedDerivation::substitutesAllowed() const
-{
-    return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true);
-}
-
-bool ParsedDerivation::useUidRange() const
-{
-    return getRequiredSystemFeatures().count("uid-range");
+    auto ss = getStringsAttr(name);
+    return ss
+        ? (std::optional{StringSet{ss->begin(), ss->end()}})
+        : (std::optional<StringSet>{});
 }
 
 static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
@@ -188,7 +153,6 @@ static nlohmann::json pathInfoToJSON(
 
 std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
 {
-    auto structuredAttrs = getStructuredAttrs();
     if (!structuredAttrs) return std::nullopt;
 
     auto json = *structuredAttrs;

src/libstore/parsed-derivations.hh

@@ -8,38 +8,40 @@
 
 namespace nix {
 
+struct DerivationOptions;
+
 class ParsedDerivation
 {
     StorePath drvPath;
     BasicDerivation & drv;
     std::unique_ptr<nlohmann::json> structuredAttrs;
 
-public:
-
-    ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv);
-
-    ~ParsedDerivation();
-
-    const nlohmann::json * getStructuredAttrs() const
-    {
-        return structuredAttrs.get();
-    }
-
     std::optional<std::string> getStringAttr(const std::string & name) const;
 
     bool getBoolAttr(const std::string & name, bool def = false) const;
 
     std::optional<Strings> getStringsAttr(const std::string & name) const;
 
-    StringSet getRequiredSystemFeatures() const;
+    std::optional<StringSet> getStringSetAttr(const std::string & name) const;
 
-    bool canBuildLocally(Store & localStore) const;
+    /**
+     * Only `DerivationOptions` is allowed to parse individual fields
+     * from `ParsedDerivation`. This ensure that it includes all
+     * derivation options, and, the likes of `LocalDerivationGoal` are
+     * incapable of more ad-hoc options.
+     */
+    friend struct DerivationOptions;
 
-    bool willBuildLocally(Store & localStore) const;
+public:
 
-    bool substitutesAllowed() const;
+    ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv);
 
-    bool useUidRange() const;
+    ~ParsedDerivation();
+
+    bool hasStructuredAttrs() const
+    {
+        return static_cast<bool>(structuredAttrs);
+    }
 
     std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
 };

src/libstore/remote-store.cc

@@ -608,7 +608,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
         auto conn(getConnection());
 
         if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) {
-            warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4");
+            warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4");
             return callback(nullptr);
         }
 

src/libstore/ssh.cc

@@ -117,10 +117,10 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
     ProcessOptions options;
     options.dieWithParent = false;
 
+    std::unique_ptr<Logger::Suspension> loggerSuspension;
     if (!fakeSSH && !useMaster) {
-        logger->pause();
+        loggerSuspension = std::make_unique<Logger::Suspension>(logger->suspend());
     }
-    Finally cleanup = [&]() { logger->resume(); };
 
     conn->sshPid = startProcess([&]() {
         restoreProcessContext();
@@ -199,8 +199,7 @@ Path SSHMaster::startMaster()
     ProcessOptions options;
     options.dieWithParent = false;
 
-    logger->pause();
-    Finally cleanup = [&]() { logger->resume(); };
+    auto suspension = logger->suspend();
 
     if (isMasterRunning())
         return state->socketPath;

src/libstore/store-api.hh

@@ -715,7 +715,7 @@ public:
 
     /**
      * Given a store path, return the realisation actually used in the realisation of this path:
-     * - If the path is a content-addressed derivation, try to resolve it
+     * - If the path is a content-addressing derivation, try to resolve it
      * - Otherwise, find one of its derivers
      */
     std::optional<StorePath> getBuildDerivationPath(const StorePath &);

src/libstore/unix/build/local-derivation-goal.cc

@@ -184,10 +184,6 @@ void LocalDerivationGoal::killSandbox(bool getStats)
 
 Goal::Co LocalDerivationGoal::tryLocalBuild()
 {
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
         worker.waitForBuildSlot(shared_from_this());
@@ -200,13 +196,12 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
 
     /* Are we doing a chroot build? */
     {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
         if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
+            if (drvOptions->noChroot)
                 throw Error("derivation '%s' has '__noChroot' set, "
                     "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
 #if __APPLE__
-            if (additionalSandboxProfile != "")
+            if (drvOptions->additionalSandboxProfile != "")
                 throw Error("derivation '%s' specifies a sandbox profile, "
                     "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
 #endif
@@ -215,7 +210,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = derivationType->isSandboxed() && !noChroot;
+            useChroot = derivationType->isSandboxed() && !drvOptions->noChroot;
     }
 
     auto & localStore = getLocalStore();
@@ -240,7 +235,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
 
     if (useBuildUsers()) {
         if (!buildUser)
-            buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
+            buildUser = acquireUserLock(drvOptions->useUidRange(*drv) ? 65536 : 1, useChroot);
 
         if (!buildUser) {
             if (!actLock)
@@ -531,13 +526,19 @@ void LocalDerivationGoal::startBuilder()
     killSandbox(false);
 
     /* Right platform? */
-    if (!parsedDrv->canBuildLocally(worker.store))
-        throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
-            drv->platform,
-            concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
-            worker.store.printStorePath(drvPath),
-            settings.thisSystem,
-            concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
+    if (!drvOptions->canBuildLocally(worker.store, *drv)) {
+        // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2
+        if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") {
+            throw Error("run `/usr/sbin/softwareupdate --install-rosetta` to enable your %s to run programs for %s", settings.thisSystem, drv->platform);
+        } else {
+            throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
+                drv->platform,
+                concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
+                worker.store.printStorePath(drvPath),
+                settings.thisSystem,
+                concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
+        }
+    }
 
     /* Create a temporary directory where the build will take
        place. */
@@ -622,7 +623,7 @@ void LocalDerivationGoal::startBuilder()
     writeStructuredAttrs();
 
     /* Handle exportReferencesGraph(), if set. */
-    if (!parsedDrv->getStructuredAttrs()) {
+    if (!parsedDrv->hasStructuredAttrs()) {
         /* The `exportReferencesGraph' feature allows the references graph
            to be passed to a builder.  This attribute should be a list of
            pairs [name1 path1 name2 path2 ...].  The references graph of
@@ -696,7 +697,7 @@ void LocalDerivationGoal::startBuilder()
         PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
         /* This works like the above, except on a per-derivation level */
-        auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
+        auto impurePaths = drvOptions->impureHostDeps;
 
         for (auto & i : impurePaths) {
             bool found = false;
@@ -716,7 +717,7 @@ void LocalDerivationGoal::startBuilder()
                 throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
                     worker.store.printStorePath(drvPath), i);
 
-            /* Allow files in __impureHostDeps to be missing; e.g.
+            /* Allow files in drvOptions->impureHostDeps to be missing; e.g.
                macOS 11+ has no /usr/lib/libSystem*.dylib */
             pathsInChroot[i] = {i, true};
         }
@@ -756,10 +757,10 @@ void LocalDerivationGoal::startBuilder()
            nobody account.  The latter is kind of a hack to support
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
-        if (parsedDrv->useUidRange())
+        if (drvOptions->useUidRange(*drv))
             chownToBuilder(chrootRootDir + "/etc");
 
-        if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
+        if (drvOptions->useUidRange(*drv) && (!buildUser || buildUser->getUIDCount() < 65536))
             throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
 
         /* Declare the build user's group so that programs get a consistent
@@ -800,7 +801,7 @@ void LocalDerivationGoal::startBuilder()
            out. */
         for (auto & i : drv->outputsAndOptPaths(worker.store)) {
             /* If the name isn't known a priori (i.e. floating
-               content-addressed derivation), the temporary location we use
+               content-addressing derivation), the temporary location we use
                should be fresh.  Freshness means it is impossible that the path
                is already in the sandbox, so we don't need to worry about
                removing it.  */
@@ -818,7 +819,7 @@ void LocalDerivationGoal::startBuilder()
         }
 
 #else
-        if (parsedDrv->useUidRange())
+        if (drvOptions->useUidRange(*drv))
             throw Error("feature 'uid-range' is not supported on this platform");
         #if __APPLE__
             /* We don't really have any parent prep work to do (yet?)
@@ -828,7 +829,7 @@ void LocalDerivationGoal::startBuilder()
         #endif
 #endif
     } else {
-        if (parsedDrv->useUidRange())
+        if (drvOptions->useUidRange(*drv))
             throw Error("feature 'uid-range' is only supported in sandboxed builds");
     }
 
@@ -873,7 +874,7 @@ void LocalDerivationGoal::startBuilder()
 
     /* Fire up a Nix daemon to process recursive Nix calls from the
        builder. */
-    if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
+    if (drvOptions->getRequiredSystemFeatures(*drv).count("recursive-nix"))
         startDaemon();
 
     /* Run the builder. */
@@ -1141,18 +1142,12 @@ void LocalDerivationGoal::initTmpDir()
     tmpDirInSandbox = tmpDir;
 #endif
 
-    /* In non-structured mode, add all bindings specified in the
-       derivation via the environment, except those listed in the
-       passAsFile attribute. Those are passed as file names pointing
-       to temporary files containing the contents. Note that
-       passAsFile is ignored in structure mode because it's not
-       needed (attributes are not passed through the environment, so
-       there is no size constraint). */
-    if (!parsedDrv->getStructuredAttrs()) {
-
-        StringSet passAsFile = tokenizeString<StringSet>(getOr(drv->env, "passAsFile", ""));
+    /* In non-structured mode, set all bindings either directory in the
+       environment or via a file, as specified by
+       `DerivationOptions::passAsFile`. */
+    if (!parsedDrv->hasStructuredAttrs()) {
         for (auto & i : drv->env) {
-            if (passAsFile.find(i.first) == passAsFile.end()) {
+            if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) {
                 env[i.first] = i.second;
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
@@ -1229,7 +1224,7 @@ void LocalDerivationGoal::initEnv()
         if (!impureEnv.empty())
             experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv);
 
-        for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) {
+        for (auto & i : drvOptions->impureEnvVars){
             auto envVar = impureEnv.find(i);
             if (envVar != impureEnv.end()) {
                 env[i] = envVar->second;
@@ -1989,7 +1984,7 @@ void LocalDerivationGoal::runChild()
             }
 
             /* Make /etc unwritable */
-            if (!parsedDrv->useUidRange())
+            if (!drvOptions->useUidRange(*drv))
                 chmod_(chrootRootDir + "/etc", 0555);
 
             /* Unshare this mount namespace. This is necessary because
@@ -2149,7 +2144,18 @@ void LocalDerivationGoal::runChild()
                without file-write* allowed, access() incorrectly returns EPERM
              */
             sandboxProfile += "(allow file-read* file-write* process-exec\n";
+
+            // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter.
+            // See https://github.com/NixOS/nix/issues/4119
+            // We split our allow groups approximately at half the actual limit, 1 << 16
+            const int breakpoint = sandboxProfile.length() + (1 << 14);
             for (auto & i : pathsInChroot) {
+
+                if (sandboxProfile.length() >= breakpoint) {
+                    debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint);
+                    sandboxProfile += ")\n(allow file-read* file-write* process-exec\n";
+                }
+
                 if (i.first != i.second.source)
                     throw Error(
                         "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
@@ -2176,7 +2182,7 @@ void LocalDerivationGoal::runChild()
             }
             sandboxProfile += ")\n";
 
-            sandboxProfile += additionalSandboxProfile;
+            sandboxProfile += drvOptions->additionalSandboxProfile;
         } else
             sandboxProfile +=
                 #include "sandbox-minimal.sb"
@@ -2185,8 +2191,6 @@ void LocalDerivationGoal::runChild()
         debug("Generated sandbox profile:");
         debug(sandboxProfile);
 
-        bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
         /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
             to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
         Path globalTmpDir = canonPath(defaultTempDir(), true);
@@ -2199,7 +2203,7 @@ void LocalDerivationGoal::runChild()
             Strings sandboxArgs;
             sandboxArgs.push_back("_GLOBAL_TMP_DIR");
             sandboxArgs.push_back(globalTmpDir);
-            if (allowLocalNetworking) {
+            if (drvOptions->allowLocalNetworking) {
                 sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING");
                 sandboxArgs.push_back("1");
             }
@@ -2219,7 +2223,7 @@ void LocalDerivationGoal::runChild()
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
             try {
-                logger = makeJSONLogger(*logger);
+                logger = makeJSONLogger(getStandardError());
 
                 std::map<std::string, Path> outputs;
                 for (auto & e : drv->outputs)
@@ -2291,7 +2295,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
        to do anything here.
 
        We can only early return when the outputs are known a priori. For
-       floating content-addressed derivations this isn't the case.
+       floating content-addressing derivations this isn't the case.
      */
     if (hook)
         return DerivationGoal::registerOutputs();
@@ -2389,14 +2393,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             inodesSeen);
 
         bool discardReferences = false;
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
-                if (auto output = get(*udr, outputName)) {
-                    if (!output->is_boolean())
-                        throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string());
-                    discardReferences = output->get<bool>();
-                }
-            }
+        if (auto udr = get(drvOptions->unsafeDiscardReferences, outputName)) {
+            discardReferences = *udr;
         }
 
         StorePathSet references;
@@ -2565,7 +2563,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 case FileIngestionMethod::Git: {
                     return git::dumpHash(
                         outputHash.hashAlgo,
-                        {getFSSourceAccessor(), CanonPath(tmpDir + "/tmp")}).hash;
+                        {getFSSourceAccessor(), CanonPath(actualPath)}).hash;
                 }
                 }
                 assert(false);
@@ -2867,13 +2865,6 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
         auto & outputName = output.first;
         auto & info = output.second;
 
-        struct Checks
-        {
-            bool ignoreSelfRefs = false;
-            std::optional<uint64_t> maxSize, maxClosureSize;
-            std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
-        };
-
         /* Compute the closure and closure size of some output. This
            is slightly tricky because some of its references (namely
            other outputs) may not be valid yet. */
@@ -2905,7 +2896,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
             return std::make_pair(std::move(pathsDone), closureSize);
         };
 
-        auto applyChecks = [&](const Checks & checks)
+        auto applyChecks = [&](const DerivationOptions::OutputChecks & checks)
         {
             if (checks.maxSize && info.narSize > *checks.maxSize)
                 throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
@@ -2918,15 +2909,13 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                         worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
             }
 
-            auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
+            auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive)
             {
-                if (!value) return;
-
                 /* Parse a list of reference specifiers.  Each element must
                    either be a store path, or the symbolic name of the output
                    of the derivation (such as `out'). */
                 StorePathSet spec;
-                for (auto & i : *value) {
+                for (auto & i : value) {
                     if (worker.store.isStorePath(i))
                         spec.insert(worker.store.parseStorePath(i));
                     else if (auto output = get(outputs, i))
@@ -2964,73 +2953,35 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                 }
             };
 
-            checkRefs(checks.allowedReferences, true, false);
-            checkRefs(checks.allowedRequisites, true, true);
-            checkRefs(checks.disallowedReferences, false, false);
-            checkRefs(checks.disallowedRequisites, false, true);
+            /* Mandatory check: absent whitelist, and present but empty
+               whitelist mean very different things. */
+            if (auto & refs = checks.allowedReferences) {
+                checkRefs(*refs, true, false);
+            }
+            if (auto & refs = checks.allowedRequisites) {
+                checkRefs(*refs, true, true);
+            }
+
+            /* Optimization: don't need to do anything when
+               disallowed and empty set. */
+            if (!checks.disallowedReferences.empty()) {
+                checkRefs(checks.disallowedReferences, false, false);
+            }
+            if (!checks.disallowedRequisites.empty()) {
+                checkRefs(checks.disallowedRequisites, false, true);
+            }
         };
 
-        if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
-            if (get(*structuredAttrs, "allowedReferences")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
-            }
-            if (get(*structuredAttrs, "allowedRequisites")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
-            }
-            if (get(*structuredAttrs, "disallowedRequisites")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
-            }
-            if (get(*structuredAttrs, "disallowedReferences")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
-            }
-            if (get(*structuredAttrs, "maxSize")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
-            }
-            if (get(*structuredAttrs, "maxClosureSize")){
-                warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
-            }
-            if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
-                if (auto output = get(*outputChecks, outputName)) {
-                    Checks checks;
+        std::visit(overloaded{
+            [&](const DerivationOptions::OutputChecks & checks) {
+                applyChecks(checks);
+            },
+            [&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
+                if (auto outputChecks = get(checksPerOutput, outputName))
 
-                    if (auto maxSize = get(*output, "maxSize"))
-                        checks.maxSize = maxSize->get<uint64_t>();
-
-                    if (auto maxClosureSize = get(*output, "maxClosureSize"))
-                        checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
-                    auto get_ = [&](const std::string & name) -> std::optional<Strings> {
-                        if (auto i = get(*output, name)) {
-                            Strings res;
-                            for (auto j = i->begin(); j != i->end(); ++j) {
-                                if (!j->is_string())
-                                    throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
-                                res.push_back(j->get<std::string>());
-                            }
-                            checks.disallowedRequisites = res;
-                            return res;
-                        }
-                        return {};
-                    };
-
-                    checks.allowedReferences = get_("allowedReferences");
-                    checks.allowedRequisites = get_("allowedRequisites");
-                    checks.disallowedReferences = get_("disallowedReferences");
-                    checks.disallowedRequisites = get_("disallowedRequisites");
-
-                    applyChecks(checks);
-                }
-            }
-        } else {
-            // legacy non-structured-attributes case
-            Checks checks;
-            checks.ignoreSelfRefs = true;
-            checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
-            checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
-            checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
-            checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
-            applyChecks(checks);
-        }
+                    applyChecks(*outputChecks);
+            },
+        }, drvOptions->outputChecks);
     }
 }
 

src/libstore/unix/build/local-derivation-goal.hh

@@ -109,11 +109,6 @@ struct LocalDerivationGoal : public DerivationGoal
     typedef map<std::string, std::string> Environment;
     Environment env;
 
-#if __APPLE__
-    typedef std::string SandboxProfile;
-    SandboxProfile additionalSandboxProfile;
-#endif
-
     /**
      * Hash rewriting.
      */
@@ -130,7 +125,7 @@ struct LocalDerivationGoal : public DerivationGoal
      *   rewrite after the build. Otherwise the regular predetermined paths are
      *   put here.
      *
-     * - Floating content-addressed derivations do not know their final build
+     * - Floating content-addressing derivations do not know their final build
      *   output paths until the outputs are hashed, so random locations are
      *   used, and then renamed. The randomness helps guard against hidden
      *   self-references.

src/libutil-tests/hash.cc

@@ -6,10 +6,52 @@
 
 namespace nix {
 
+class BLAKE3HashTest : public virtual ::testing::Test
+{
+public:
+
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
+
+private:
+
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "blake3-hashes");
+    }
+};
+
     /* ----------------------------------------------------------------------------
      * hashString
      * --------------------------------------------------------------------------*/
 
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes1) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abc";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85");
+    }
+
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes2) {
+        // values taken from: https://tools.ietf.org/html/rfc4634
+        auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8");
+    }
+
+    TEST_F(BLAKE3HashTest, testKnownBLAKE3Hashes3) {
+        // values taken from: https://www.ietf.org/archive/id/draft-aumasson-blake3-00.txt
+        auto s = "IETF";
+        auto hash = hashString(HashAlgorithm::BLAKE3, s, mockXpSettings);
+        ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
+                "blake3:83a2de1ee6f4e6ab686889248f4ec0cf4cc5709446a682ffd1cbb4d6165181e2");
+    }
+
     TEST(hashString, testKnownMD5Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";

src/libutil-tests/monitorfdhup.cc

@@ -0,0 +1,18 @@
+#include "util.hh"
+#include "monitor-fd.hh"
+
+#include <sys/file.h>
+#include <gtest/gtest.h>
+
+namespace nix {
+TEST(MonitorFdHup, shouldNotBlock)
+{
+    Pipe p;
+    p.create();
+    {
+        // when monitor gets destroyed it should cancel the
+        // background thread and do not block
+        MonitorFdHup monitor(p.readSide.get());
+    }
+}
+}

src/libutil/current-process.cc

@@ -19,10 +19,6 @@
 # include "namespaces.hh"
 #endif
 
-#ifndef _WIN32
-# include <sys/resource.h>
-#endif
-
 namespace nix {
 
 unsigned int getMaxCPU()
@@ -55,11 +51,11 @@ unsigned int getMaxCPU()
 //////////////////////////////////////////////////////////////////////
 
 
+#ifndef _WIN32
 size_t savedStackSize = 0;
 
 void setStackSize(size_t stackSize)
 {
-    #ifndef _WIN32
     struct rlimit limit;
     if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
         savedStackSize = limit.rlim_cur;
@@ -77,31 +73,8 @@ void setStackSize(size_t stackSize)
             );
         }
     }
-    #else
-    ULONG_PTR stackLow, stackHigh;
-    GetCurrentThreadStackLimits(&stackLow, &stackHigh);
-    ULONG maxStackSize = stackHigh - stackLow;
-    ULONG currStackSize = 0;
-    // This retrieves the current promised stack size
-    SetThreadStackGuarantee(&currStackSize);
-    if (currStackSize < stackSize) {
-        savedStackSize = currStackSize;
-        ULONG newStackSize = std::min(static_cast<ULONG>(stackSize), maxStackSize);
-        if (SetThreadStackGuarantee(&newStackSize) == 0) {
-            logger->log(
-                lvlError,
-                HintFmt(
-                    "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%",
-                    savedStackSize,
-                    stackSize,
-                    maxStackSize,
-                    std::to_string(GetLastError())
-                ).str()
-            );
-        }
-    }
-    #endif
 }
+#endif
 
 void restoreProcessContext(bool restoreMounts)
 {

src/libutil/current-process.hh

@@ -17,10 +17,13 @@ namespace nix {
  */
 unsigned int getMaxCPU();
 
+// It does not seem possible to dynamically change stack size on Windows.
+#ifndef _WIN32
 /**
  * Change the stack size.
  */
 void setStackSize(size_t stackSize);
+#endif
 
 /**
  * Restore the original inherited Unix process context (such as signal

src/libutil/error.hh

@@ -50,6 +50,14 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
+/* NOTE: position.hh recursively depends on source-path.hh -> source-accessor.hh
+   -> hash.hh -> config.hh -> experimental-features.hh -> error.hh -> Pos.
+   There are other such cycles.
+   Thus, Pos has to be an incomplete type in this header. But since ErrorInfo/Trace
+   have to refer to Pos, they have to use pointer indirection via std::shared_ptr
+   to break the recursive header dependency.
+   FIXME: Untangle this mess. Should there be AbstractPos as there used to be before
+   4feb7d9f71? */
 struct Pos;
 
 void printCodeLines(std::ostream & out,

src/libutil/experimental-features.cc

@@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails
  * feature, we either have no issue at all if few features are not added
  * at the end of the list, or a proper merge conflict if they are.
  */
-constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::PipeOperators);
+constexpr size_t numXpFeatures = 1 + static_cast<size_t>(Xp::BLAKE3Hashes);
 
 constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails = {{
     {
@@ -109,6 +109,8 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
 
             runCommand "foo"
               {
+                 # Optional: let Nix know "foo" requires the experimental feature
+                 requiredSystemFeatures = [ "recursive-nix" ];
                  buildInputs = [ nix jq ];
                  NIX_PATH = "nixpkgs=${<nixpkgs>}";
               }
@@ -286,6 +288,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         )",
         .trackingUrl = "https://github.com/NixOS/nix/milestone/55",
     },
+    {
+        .tag = Xp::BLAKE3Hashes,
+        .name = "blake3-hashes",
+        .description = R"(
+            Enables support for BLAKE3 hashes.
+        )",
+        .trackingUrl = "",
+    },
 }};
 
 static_assert(

src/libutil/experimental-features.hh

@@ -35,6 +35,7 @@ enum struct ExperimentalFeature
     MountedSSHStore,
     VerifiedFetches,
     PipeOperators,
+    BLAKE3Hashes,
 };
 
 extern std::set<std::string> stabilizedFeatures;

src/libutil/file-descriptor.cc

@@ -1,6 +1,3 @@
-#include "file-system.hh"
-#include "signals.hh"
-#include "finally.hh"
 #include "serialise.hh"
 #include "util.hh"
 

src/libutil/hash.cc

@@ -1,6 +1,7 @@
 #include <iostream>
 #include <cstring>
 
+#include <blake3.h>
 #include <openssl/crypto.h>
 #include <openssl/md5.h>
 #include <openssl/sha.h>
@@ -8,6 +9,7 @@
 #include "args.hh"
 #include "hash.hh"
 #include "archive.hh"
+#include "config.hh"
 #include "split.hh"
 
 #include <sys/types.h>
@@ -20,6 +22,7 @@ namespace nix {
 
 static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
+    case HashAlgorithm::BLAKE3: return blake3HashSize;
     case HashAlgorithm::MD5: return md5HashSize;
     case HashAlgorithm::SHA1: return sha1HashSize;
     case HashAlgorithm::SHA256: return sha256HashSize;
@@ -29,12 +32,15 @@ static size_t regularHashSize(HashAlgorithm type) {
 }
 
 
-const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512" };
 
 const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
-Hash::Hash(HashAlgorithm algo) : algo(algo)
+Hash::Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings) : algo(algo)
 {
+    if (algo == HashAlgorithm::BLAKE3) {
+        xpSettings.require(Xp::BLAKE3Hashes);
+    }
     hashSize = regularHashSize(algo);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
@@ -284,6 +290,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha
 
 union Ctx
 {
+    blake3_hasher blake3;
     MD5_CTX md5;
     SHA_CTX sha1;
     SHA256_CTX sha256;
@@ -293,7 +300,8 @@ union Ctx
 
 static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_init(&ctx.blake3);
+    else if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
@@ -303,7 +311,8 @@ static void start(HashAlgorithm ha, Ctx & ctx)
 static void update(HashAlgorithm ha, Ctx & ctx,
                    std::string_view data)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_update(&ctx.blake3, data.data(), data.size());
+    else if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
     else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
@@ -312,24 +321,24 @@ static void update(HashAlgorithm ha, Ctx & ctx,
 
 static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    if (ha == HashAlgorithm::BLAKE3) blake3_hasher_finalize(&ctx.blake3, hash, BLAKE3_OUT_LEN);
+    else if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
     else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
     else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
     else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
 }
 
-
-Hash hashString(HashAlgorithm ha, std::string_view s)
+Hash hashString(
+    HashAlgorithm ha, std::string_view s, const ExperimentalFeatureSettings & xpSettings)
 {
     Ctx ctx;
-    Hash hash(ha);
+    Hash hash(ha, xpSettings);
     start(ha, ctx);
     update(ha, ctx, s);
     finish(ha, ctx, hash.hash);
     return hash;
 }
 
-
 Hash hashFile(HashAlgorithm ha, const Path & path)
 {
     HashSink sink(ha);
@@ -426,6 +435,7 @@ std::string_view printHashFormat(HashFormat HashFormat)
 
 std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
+    if (s == "blake3") return HashAlgorithm::BLAKE3;
     if (s == "md5") return HashAlgorithm::MD5;
     if (s == "sha1") return HashAlgorithm::SHA1;
     if (s == "sha256") return HashAlgorithm::SHA256;
@@ -439,12 +449,13 @@ HashAlgorithm parseHashAlgo(std::string_view s)
     if (opt_h)
         return *opt_h;
     else
-        throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
+        throw UsageError("unknown hash algorithm '%1%', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
 std::string_view printHashAlgo(HashAlgorithm ha)
 {
     switch (ha) {
+    case HashAlgorithm::BLAKE3: return "blake3";
     case HashAlgorithm::MD5: return "md5";
     case HashAlgorithm::SHA1: return "sha1";
     case HashAlgorithm::SHA256: return "sha256";

src/libutil/hash.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "config.hh"
 #include "types.hh"
 #include "serialise.hh"
 #include "file-system.hh"
@@ -11,9 +12,9 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
-
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512, BLAKE3 };
 
+const int blake3HashSize = 32;
 const int md5HashSize = 16;
 const int sha1HashSize = 20;
 const int sha256HashSize = 32;
@@ -52,7 +53,7 @@ struct Hash
     /**
      * Create a zero-filled hash object.
      */
-    explicit Hash(HashAlgorithm algo);
+    explicit Hash(HashAlgorithm algo, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Parse the hash from a string representation in the format
@@ -157,7 +158,7 @@ std::string printHash16or32(const Hash & hash);
 /**
  * Compute the hash of the given string.
  */
-Hash hashString(HashAlgorithm ha, std::string_view s);
+Hash hashString(HashAlgorithm ha, std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
 /**
  * Compute the hash of the given file, hashing its contents directly.

src/libutil/json-utils.cc

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include <nlohmann/json_fwd.hpp>
 #include <iostream>
+#include <optional>
 
 namespace nix {
 
@@ -38,6 +39,15 @@ std::optional<nlohmann::json> optionalValueAt(const nlohmann::json::object_t & m
     return std::optional { map.at(key) };
 }
 
+std::optional<nlohmann::json> nullableValueAt(const nlohmann::json::object_t & map, const std::string & key)
+{
+    auto value = valueAt(map, key);
+
+    if (value.is_null())
+        return std::nullopt;
+
+    return std::optional { std::move(value) };
+}
 
 const nlohmann::json * getNullable(const nlohmann::json & value)
 {

src/libutil/json-utils.hh

@@ -25,6 +25,7 @@ const nlohmann::json & valueAt(
     const std::string & key);
 
 std::optional<nlohmann::json> optionalValueAt(const nlohmann::json::object_t & value, const std::string & key);
+std::optional<nlohmann::json> nullableValueAt(const nlohmann::json::object_t & value, const std::string & key);
 
 /**
  * Downcast the json object, failing with a nice error if the conversion fails.
@@ -69,6 +70,9 @@ struct json_avoids_null<std::vector<T>> : std::true_type {};
 template<typename T>
 struct json_avoids_null<std::list<T>> : std::true_type {};
 
+template<typename T>
+struct json_avoids_null<std::set<T>> : std::true_type {};
+
 template<typename K, typename V>
 struct json_avoids_null<std::map<K, V>> : std::true_type {};
 

src/libutil/logging.cc

@@ -29,7 +29,7 @@ void setCurActivity(const ActivityId activityId)
     curActivity = activityId;
 }
 
-Logger * logger = makeSimpleLogger(true);
+std::unique_ptr<Logger> logger = makeSimpleLogger(true);
 
 void Logger::warn(const std::string & msg)
 {
@@ -43,6 +43,19 @@ void Logger::writeToStdout(std::string_view s)
     writeFull(standard_out, "\n");
 }
 
+Logger::Suspension Logger::suspend()
+{
+    pause();
+    return Suspension { ._finalize = {[this](){this->resume();}} };
+}
+
+std::optional<Logger::Suspension> Logger::suspendIf(bool cond)
+{
+    if (cond)
+        return suspend();
+    return {};
+}
+
 class SimpleLogger : public Logger
 {
 public:
@@ -128,9 +141,9 @@ void writeToStderr(std::string_view s)
     }
 }
 
-Logger * makeSimpleLogger(bool printBuildLogs)
+std::unique_ptr<Logger> makeSimpleLogger(bool printBuildLogs)
 {
-    return new SimpleLogger(printBuildLogs);
+    return std::make_unique<SimpleLogger>(printBuildLogs);
 }
 
 std::atomic<uint64_t> nextId{0};
@@ -167,9 +180,9 @@ void to_json(nlohmann::json & json, std::shared_ptr<Pos> pos)
 }
 
 struct JSONLogger : Logger {
-    Logger & prevLogger;
+    Descriptor fd;
 
-    JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { }
+    JSONLogger(Descriptor fd) : fd(fd) { }
 
     bool isVerbose() override {
         return true;
@@ -190,7 +203,7 @@ struct JSONLogger : Logger {
 
     void write(const nlohmann::json & json)
     {
-        prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace));
+        writeLine(fd, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace));
     }
 
     void log(Verbosity lvl, std::string_view s) override
@@ -262,9 +275,9 @@ struct JSONLogger : Logger {
     }
 };
 
-Logger * makeJSONLogger(Logger & prevLogger)
+std::unique_ptr<Logger> makeJSONLogger(Descriptor fd)
 {
-    return new JSONLogger(prevLogger);
+    return std::make_unique<JSONLogger>(fd);
 }
 
 static Logger::Fields getFields(nlohmann::json & json)

src/libutil/logging.hh

@@ -3,6 +3,8 @@
 
 #include "error.hh"
 #include "config.hh"
+#include "file-descriptor.hh"
+#include "finally.hh"
 
 #include <nlohmann/json_fwd.hpp>
 
@@ -74,6 +76,17 @@ public:
 
     virtual void stop() { };
 
+    /**
+     * Guard object to resume the logger when done.
+     */
+    struct Suspension {
+        Finally<std::function<void()>> _finalize;
+    };
+
+    Suspension suspend();
+
+    std::optional<Suspension> suspendIf(bool cond);
+
     virtual void pause() { };
     virtual void resume() { };
 
@@ -179,11 +192,11 @@ struct PushActivity
     ~PushActivity() { setCurActivity(prevAct); }
 };
 
-extern Logger * logger;
+extern std::unique_ptr<Logger> logger;
 
-Logger * makeSimpleLogger(bool printBuildLogs = true);
+std::unique_ptr<Logger> makeSimpleLogger(bool printBuildLogs = true);
 
-Logger * makeJSONLogger(Logger & prevLogger);
+std::unique_ptr<Logger> makeJSONLogger(Descriptor fd);
 
 /**
  * @param source A noun phrase describing the source of the message, e.g. "the builder".

src/libutil/meson.build

@@ -62,6 +62,12 @@ elif host_machine.system() == 'sunos'
   deps_other += [socket, network_service_library]
 endif
 
+blake3 = dependency(
+  'libblake3',
+  version: '>= 1.5.5',
+)
+deps_private += blake3
+
 boost = dependency(
   'boost',
   modules : ['context', 'coroutine'],
@@ -147,7 +153,9 @@ sources = files(
   'json-utils.cc',
   'logging.cc',
   'memory-source-accessor.cc',
+  'mounted-source-accessor.cc',
   'position.cc',
+  'pos-table.cc',
   'posix-source-accessor.cc',
   'references.cc',
   'serialise.cc',
@@ -160,6 +168,7 @@ sources = files(
   'tarfile.cc',
   'terminal.cc',
   'thread-pool.cc',
+  'union-source-accessor.cc',
   'unix-domain-socket.cc',
   'url.cc',
   'users.cc',
@@ -217,6 +226,8 @@ headers = [config_h] + files(
   'muxable-pipe.hh',
   'os-string.hh',
   'pool.hh',
+  'pos-idx.hh',
+  'pos-table.hh',
   'position.hh',
   'posix-source-accessor.hh',
   'processes.hh',

src/libutil/mounted-source-accessor.cc

@@ -1,4 +1,4 @@
-#include "mounted-source-accessor.hh"
+#include "source-accessor.hh"
 
 namespace nix {
 
@@ -23,12 +23,6 @@ struct MountedSourceAccessor : SourceAccessor
         return accessor->readFile(subpath);
     }
 
-    bool pathExists(const CanonPath & path) override
-    {
-        auto [accessor, subpath] = resolve(path);
-        return accessor->pathExists(subpath);
-    }
-
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
         auto [accessor, subpath] = resolve(path);
@@ -69,6 +63,12 @@ struct MountedSourceAccessor : SourceAccessor
             path.pop();
         }
     }
+
+    std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->getPhysicalPath(subpath);
+    }
 };
 
 ref<SourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)

src/libutil/package.nix

@@ -6,6 +6,7 @@
   boost,
   brotli,
   libarchive,
+  libblake3,
   libcpuid,
   libsodium,
   nlohmann_json,
@@ -42,6 +43,7 @@ mkMesonLibrary (finalAttrs: {
 
   buildInputs = [
     brotli
+    libblake3
     libsodium
     openssl
   ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;

src/libutil/pos-idx.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <cinttypes>
 #include <functional>

src/libutil/pos-table.cc

@@ -0,0 +1,37 @@
+#include "pos-table.hh"
+
+#include <algorithm>
+
+namespace nix {
+
+/* Position table. */
+
+Pos PosTable::operator[](PosIdx p) const
+{
+    auto origin = resolve(p);
+    if (!origin)
+        return {};
+
+    const auto offset = origin->offsetOf(p);
+
+    Pos result{0, 0, origin->origin};
+    auto lines = this->lines.lock();
+    auto linesForInput = (*lines)[origin->offset];
+
+    if (linesForInput.empty()) {
+        auto source = result.getSource().value_or("");
+        const char * begin = source.data();
+        for (Pos::LinesIterator it(source), end; it != end; it++)
+            linesForInput.push_back(it->data() - begin);
+        if (linesForInput.empty())
+            linesForInput.push_back(0);
+    }
+    // as above: the first line starts at byte 0 and is always present
+    auto lineStartOffset = std::prev(std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
+
+    result.line = 1 + (lineStartOffset - linesForInput.begin());
+    result.column = 1 + (offset - *lineStartOffset);
+    return result;
+}
+
+}

src/libutil/pos-table.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <cstdint>
 #include <vector>
@@ -18,9 +19,12 @@ public:
     private:
         uint32_t offset;
 
-        Origin(Pos::Origin origin, uint32_t offset, size_t size):
-            offset(offset), origin(origin), size(size)
-        {}
+        Origin(Pos::Origin origin, uint32_t offset, size_t size)
+            : offset(offset)
+            , origin(origin)
+            , size(size)
+        {
+        }
 
     public:
         const Pos::Origin origin;
@@ -72,6 +76,17 @@ public:
         return PosIdx(1 + origin.offset + offset);
     }
 
+    /**
+     * Convert a byte-offset PosIdx into a Pos with line/column information.
+     *
+     * @param p Byte offset into the virtual concatenation of all parsed contents
+     * @return Position
+     *
+     * @warning Very expensive to call, as this has to read the entire source
+     * into memory each time. Call this only if absolutely necessary. Prefer
+     * to keep PosIdx around instead of needlessly converting it into Pos by
+     * using this lookup method.
+     */
     Pos operator[](PosIdx p) const;
 
     Pos::Origin originOf(PosIdx p) const

src/libutil/position.cc

@@ -66,6 +66,13 @@ std::optional<std::string> Pos::getSource() const
     }, origin);
 }
 
+std::optional<SourcePath> Pos::getSourcePath() const
+{
+    if (auto * path = std::get_if<SourcePath>(&origin))
+        return *path;
+    return std::nullopt;
+}
+
 void Pos::print(std::ostream & out, bool showOrigin) const
 {
     if (showOrigin) {

src/libutil/position.hh

@@ -50,6 +50,7 @@ struct Pos
 
     explicit operator bool() const { return line > 0; }
 
+    /* TODO: Why std::shared_ptr<Pos> and not std::shared_ptr<const Pos>? */
     operator std::shared_ptr<Pos>() const;
 
     /**
@@ -69,9 +70,7 @@ struct Pos
     /**
      * Get the SourcePath, if the source was loaded from a file.
      */
-    std::optional<SourcePath> getSourcePath() const {
-        return *std::get_if<SourcePath>(&origin);
-    }
+    std::optional<SourcePath> getSourcePath() const;
 
     struct LinesIterator {
         using difference_type = size_t;

src/libutil/serialise.cc

@@ -227,8 +227,7 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
                                 throw EndOfFile("coroutine has finished");
                         }
 
-                        size_t n = std::min(cur.size(), out_len);
-                        memcpy(out, cur.data(), n);
+                        size_t n = cur.copy(out, out_len);
                         cur.remove_prefix(n);
                         return n;
                     });
@@ -260,7 +259,7 @@ std::unique_ptr<Source> sinkToSource(
 {
     struct SinkToSource : Source
     {
-        typedef boost::coroutines2::coroutine<std::string> coro_t;
+        typedef boost::coroutines2::coroutine<std::string_view> coro_t;
 
         std::function<void(Sink &)> fun;
         std::function<void()> eof;
@@ -271,33 +270,37 @@ std::unique_ptr<Source> sinkToSource(
         {
         }
 
-        std::string cur;
-        size_t pos = 0;
+        std::string_view cur;
 
         size_t read(char * data, size_t len) override
         {
-            if (!coro) {
+            bool hasCoro = coro.has_value();
+            if (!hasCoro) {
                 coro = coro_t::pull_type([&](coro_t::push_type & yield) {
                     LambdaSink sink([&](std::string_view data) {
-                        if (!data.empty()) yield(std::string(data));
+                        if (!data.empty()) {
+                            yield(data);
+                        }
                     });
                     fun(sink);
                 });
             }
 
-            if (!*coro) { eof(); unreachable(); }
-
-            if (pos == cur.size()) {
-                if (!cur.empty()) {
+            if (cur.empty()) {
+                if (hasCoro) {
                     (*coro)();
                 }
-                cur = coro->get();
-                pos = 0;
+                if (*coro) {
+                    cur = coro->get();
+                } else {
+                    coro.reset();
+                    eof();
+                    unreachable();
+                }
             }
 
-            auto n = std::min(cur.size() - pos, len);
-            memcpy(data, cur.data() + pos, n);
-            pos += n;
+            size_t n = cur.copy(data, len);
+            cur.remove_prefix(n);
 
             return n;
         }

src/libutil/source-accessor.hh

@@ -214,4 +214,12 @@ ref<SourceAccessor> getFSSourceAccessor();
  */
 ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root);
 
+ref<SourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);
+
+/**
+ * Construct an accessor that presents a "union" view of a vector of
+ * underlying accessors. Earlier accessors take precedence over later.
+ */
+ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors);
+
 }

src/libutil/union-source-accessor.cc

@@ -0,0 +1,82 @@
+#include "source-accessor.hh"
+
+namespace nix {
+
+struct UnionSourceAccessor : SourceAccessor
+{
+    std::vector<ref<SourceAccessor>> accessors;
+
+    UnionSourceAccessor(std::vector<ref<SourceAccessor>> _accessors)
+        : accessors(std::move(_accessors))
+    {
+        displayPrefix.clear();
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors) {
+            auto st = accessor->maybeLstat(path);
+            if (st)
+                return accessor->readFile(path);
+        }
+        throw FileNotFound("path '%s' does not exist", showPath(path));
+    }
+
+    std::optional<Stat> maybeLstat(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors) {
+            auto st = accessor->maybeLstat(path);
+            if (st)
+                return st;
+        }
+        return std::nullopt;
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        DirEntries result;
+        for (auto & accessor : accessors) {
+            auto st = accessor->maybeLstat(path);
+            if (!st)
+                continue;
+            for (auto & entry : accessor->readDirectory(path))
+                // Don't override entries from previous accessors.
+                result.insert(entry);
+        }
+        return result;
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors) {
+            auto st = accessor->maybeLstat(path);
+            if (st)
+                return accessor->readLink(path);
+        }
+        throw FileNotFound("path '%s' does not exist", showPath(path));
+    }
+
+    std::string showPath(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors)
+            return accessor->showPath(path);
+        return SourceAccessor::showPath(path);
+    }
+
+    std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
+    {
+        for (auto & accessor : accessors) {
+            auto p = accessor->getPhysicalPath(path);
+            if (p)
+                return p;
+        }
+        return std::nullopt;
+    }
+};
+
+ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors)
+{
+    return make_ref<UnionSourceAccessor>(std::move(accessors));
+}
+
+}

src/libutil/unix/file-descriptor.cc

@@ -5,9 +5,27 @@
 
 #include <fcntl.h>
 #include <unistd.h>
+#include <poll.h>
 
 namespace nix {
 
+namespace {
+
+// This function is needed to handle non-blocking reads/writes. This is needed in the buildhook, because
+// somehow the json logger file descriptor ends up beeing non-blocking and breaks remote-building.
+// TODO: get rid of buildhook and remove this function again (https://github.com/NixOS/nix/issues/12688)
+void pollFD(int fd, int events)
+{
+    struct pollfd pfd;
+    pfd.fd = fd;
+    pfd.events = events;
+    int ret = poll(&pfd, 1, -1);
+    if (ret == -1) {
+        throw SysError("poll on file descriptor failed");
+    }
+}
+}
+
 std::string readFile(int fd)
 {
     struct stat st;
@@ -17,14 +35,18 @@ std::string readFile(int fd)
     return drainFD(fd, true, st.st_size);
 }
 
-
 void readFull(int fd, char * buf, size_t count)
 {
     while (count) {
         checkInterrupt();
         ssize_t res = read(fd, buf, count);
         if (res == -1) {
-            if (errno == EINTR) continue;
+            switch (errno) {
+            case EINTR: continue;
+            case EAGAIN:
+                pollFD(fd, POLLIN);
+                continue;
+            }
             throw SysError("reading from file");
         }
         if (res == 0) throw EndOfFile("unexpected end-of-file");
@@ -39,8 +61,15 @@ void writeFull(int fd, std::string_view s, bool allowInterrupts)
     while (!s.empty()) {
         if (allowInterrupts) checkInterrupt();
         ssize_t res = write(fd, s.data(), s.size());
-        if (res == -1 && errno != EINTR)
+        if (res == -1) {
+            switch (errno) {
+            case EINTR: continue;
+            case EAGAIN:
+                pollFD(fd, POLLOUT);
+                continue;
+            }
             throw SysError("writing to file");
+        }
         if (res > 0)
             s.remove_prefix(res);
     }
@@ -56,8 +85,15 @@ std::string readLine(int fd, bool eofOk)
         // FIXME: inefficient
         ssize_t rd = read(fd, &ch, 1);
         if (rd == -1) {
-            if (errno != EINTR)
+            switch (errno) {
+            case EINTR: continue;
+            case EAGAIN: {
+                pollFD(fd, POLLIN);
+                continue;
+            }
+            default:
                 throw SysError("reading a line");
+            }
         } else if (rd == 0) {
             if (eofOk)
                 return s;

src/libutil/unix/monitor-fd.hh

@@ -14,35 +14,74 @@
 
 namespace nix {
 
-
 class MonitorFdHup
 {
 private:
     std::thread thread;
+    Pipe notifyPipe;
 
 public:
     MonitorFdHup(int fd)
     {
-        thread = std::thread([fd]() {
+        notifyPipe.create();
+        thread = std::thread([this, fd]() {
             while (true) {
-                /* Wait indefinitely until a POLLHUP occurs. */
-                struct pollfd fds[1];
-                fds[0].fd = fd;
-                /* Polling for no specific events (i.e. just waiting
-                   for an error/hangup) doesn't work on macOS
-                   anymore. So wait for read events and ignore
-                   them. */
-                fds[0].events =
-                    #ifdef __APPLE__
-                    POLLRDNORM
-                    #else
+                // There is a POSIX violation on macOS: you have to listen for
+                // at least POLLHUP to receive HUP events for a FD. POSIX says
+                // this is not so, and you should just receive them regardless.
+                // However, as of our testing on macOS 14.5, the events do not
+                // get delivered if in the all-bits-unset case, but do get
+                // delivered if `POLLHUP` is set.
+                //
+                // This bug filed as rdar://37537852
+                // (https://openradar.appspot.com/37537852).
+                //
+                // macOS's own man page
+                // (https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/poll.2.html)
+                // additionally says that `POLLHUP` is ignored as an input. It
+                // seems the likely order of events here was
+                //
+                // 1. macOS did not follow the POSIX spec
+                //
+                // 2. Somebody ninja-fixed this other spec violation to make
+                // sure `POLLHUP` was not forgotten about, even though they
+                // "fixed" this issue in a spec-non-compliant way. Whatever,
+                // we'll use the fix.
+                //
+                // Relevant code, current version, which shows the :
+                // https://github.com/apple-oss-distributions/xnu/blob/94d3b452840153a99b38a3a9659680b2a006908e/bsd/kern/sys_generic.c#L1751-L1758
+                //
+                // The `POLLHUP` detection was added in
+                // https://github.com/apple-oss-distributions/xnu/commit/e13b1fa57645afc8a7b2e7d868fe9845c6b08c40#diff-a5aa0b0e7f4d866ca417f60702689fc797e9cdfe33b601b05ccf43086c35d395R1468
+                // That means added in 2007 or earlier. Should be good enough
+                // for us.
+                short hangup_events =
+#ifdef __APPLE__
+                    POLLHUP
+#else
                     0
-                    #endif
+#endif
                     ;
-                auto count = poll(fds, 1, -1);
-                if (count == -1)
-                    unreachable();
 
+                /* Wait indefinitely until a POLLHUP occurs. */
+                constexpr size_t num_fds = 2;
+                struct pollfd fds[num_fds] = {
+                    {
+                        .fd = fd,
+                        .events = hangup_events,
+                    },
+                    {
+                        .fd = notifyPipe.readSide.get(),
+                        .events = hangup_events,
+                    },
+                };
+
+                auto count = poll(fds, num_fds, -1);
+                if (count == -1) {
+                    if (errno == EINTR || errno == EAGAIN)
+                        continue;
+                    throw SysError("failed to poll() in MonitorFdHup");
+                }
                 /* This shouldn't happen, but can on macOS due to a bug.
                    See rdar://37550628.
 
@@ -50,25 +89,42 @@ public:
                    coordination with the main thread if spinning proves
                    too harmful.
                 */
-                if (count == 0) continue;
+                if (count == 0)
+                    continue;
                 if (fds[0].revents & POLLHUP) {
                     unix::triggerInterrupt();
                     break;
                 }
-                /* This will only happen on macOS. We sleep a bit to
-                   avoid waking up too often if the client is sending
-                   input. */
-                sleep(1);
+                if (fds[1].revents & POLLHUP) {
+                    break;
+                }
+                // On macOS, (jade thinks that) it is possible (although not
+                // observed on macOS 14.5) that in some limited cases on buggy
+                // kernel versions, all the non-POLLHUP events for the socket
+                // get delivered.
+                //
+                // We could sleep to avoid pointlessly spinning a thread on
+                // those, but this opens up a different problem, which is that
+                // if do sleep, it will be longer before the daemon fork for a
+                // client exits. Imagine a sequential shell script, running Nix
+                // commands, each of which talk to the daemon. If the previous
+                // command registered a temp root, exits, and then the next
+                // command issues a delete request before the temp root is
+                // cleaned up, that delete request might fail.
+                //
+                // Not sleeping doesn't actually fix the race condition --- we
+                // would need to block on the old connections' tempt roots being
+                // cleaned up in in the new connection --- but it does make it
+                // much less likely.
             }
         });
     };
 
     ~MonitorFdHup()
     {
-        pthread_cancel(thread.native_handle());
+        notifyPipe.writeSide.close();
         thread.join();
     }
 };
 
-
 }

src/libutil/unix/processes.cc

@@ -200,8 +200,15 @@ static int childEntry(void * arg)
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 {
     ChildWrapperFunction wrapper = [&] {
-        if (!options.allowVfork)
+        if (!options.allowVfork) {
+            /* Set a simple logger, while releasing (not destroying)
+               the parent logger. We don't want to run the parent
+               logger's destructor since that will crash (e.g. when
+               ~ProgressBar() tries to join a thread that doesn't
+               exist. */
+            logger.release();
             logger = makeSimpleLogger();
+        }
         try {
 #if __linux__
             if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
@@ -299,15 +306,7 @@ void runProgram2(const RunOptions & options)
     // case), so we can't use it if we alter the environment
     processOptions.allowVfork = !options.environment;
 
-    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
-    if (options.isInteractive) {
-        logger->pause();
-        resumeLoggerDefer.emplace(
-            []() {
-                logger->resume();
-            }
-        );
-    }
+    auto suspension = logger->suspendIf(options.isInteractive);
 
     /* Fork. */
     Pid pid = startProcess([&] {

src/libutil/windows/processes.cc

@@ -312,11 +312,7 @@ void runProgram2(const RunOptions & options)
     // TODO: Implement shebang / program interpreter lookup on Windows
     auto interpreter = getProgramInterpreter(realProgram);
 
-    std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
-    if (options.isInteractive) {
-        logger->pause();
-        resumeLoggerDefer.emplace([]() { logger->resume(); });
-    }
+    auto suspension = logger->suspendIf(options.isInteractive);
 
     Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in);