Merge remote-tracking branch 'origin/2.29-maintenance' into detsys-main

src/build-remote/build-remote.cc

@@ -16,7 +16,7 @@
 #include "nix/store/globals.hh"
 #include "nix/util/serialise.hh"
 #include "nix/store/build-result.hh"
-#include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/util/strings.hh"
 #include "nix/store/derivations.hh"
 #include "nix/store/local-store.hh"
@@ -42,9 +42,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
     return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
 }
 
-static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
+static bool allSupportedLocally(Store & store, const StringSet& requiredFeatures) {
     for (auto & feature : requiredFeatures)
-        if (!store.systemFeatures.get().count(feature)) return false;
+        if (!store.config.systemFeatures.get().count(feature)) return false;
     return true;
 }
 
@@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv)
            that gets cleared on reboot, but it wouldn't work on macOS. */
         auto currentLoadName = "/current-load";
         if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
-            currentLoad = std::string { localStore->stateDir } + currentLoadName;
+            currentLoad = std::string { localStore->config.stateDir } + currentLoadName;
         else
             currentLoad = settings.nixStateDir + currentLoadName;
 
@@ -113,7 +113,7 @@ static int main_build_remote(int argc, char * * argv)
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
             drvPath = store->parseStorePath(readString(source));
-            auto requiredFeatures = readStrings<std::set<std::string>>(source);
+            auto requiredFeatures = readStrings<StringSet>(source);
 
             /* It would be possible to build locally after some builds clear out,
                so don't show the warning now: */

src/libcmd/command.cc

@@ -3,7 +3,7 @@
 
 #include "nix/cmd/command.hh"
 #include "nix/cmd/markdown.hh"
-#include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/local-fs-store.hh"
 #include "nix/store/derivations.hh"
 #include "nix/expr/nixexpr.hh"
@@ -14,12 +14,10 @@
 
 namespace nix {
 
-RegisterCommand::Commands * RegisterCommand::commands = nullptr;
-
 nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
 {
     nix::Commands res;
-    for (auto & [name, command] : *RegisterCommand::commands)
+    for (auto & [name, command] : RegisterCommand::commands())
         if (name.size() == prefix.size() + 1) {
             bool equal = true;
             for (size_t i = 0; i < prefix.size(); ++i)
@@ -40,7 +38,7 @@ nlohmann::json NixMultiCommand::toJSON()
 void NixMultiCommand::run()
 {
     if (!command) {
-        std::set<std::string> subCommandTextLines;
+        StringSet subCommandTextLines;
         for (auto & [name, _] : commands)
             subCommandTextLines.insert(fmt("- `%s`", name));
         std::string markdownError =
@@ -397,4 +395,11 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
     }
 }
 
+void MixOutLinkBase::createOutLinksMaybe(const std::vector<BuiltPathWithResult> & buildables, ref<Store> & store)
+{
+    if (outLink != "")
+        if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+            createOutLinks(outLink, toBuiltPaths(buildables), *store2);
+}
+
 }

src/libcmd/common-eval-args.cc

@@ -9,7 +9,7 @@
 #include "nix/fetchers/registry.hh"
 #include "nix/flake/flakeref.hh"
 #include "nix/flake/settings.hh"
-#include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/cmd/command.hh"
 #include "nix/fetchers/tarball.hh"
 #include "nix/fetchers/fetch-to-store.hh"
@@ -18,7 +18,6 @@
 
 namespace nix {
 
-namespace fs { using namespace std::filesystem; }
 
 fetchers::Settings fetchSettings;
 
@@ -122,8 +121,8 @@ MixEvalArgs::MixEvalArgs()
         .category = category,
         .labels = {"original-ref", "resolved-ref"},
         .handler = {[&](std::string _from, std::string _to) {
-            auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string());
-            auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string());
+            auto from = parseFlakeRef(fetchSettings, _from, std::filesystem::current_path().string());
+            auto to = parseFlakeRef(fetchSettings, _to, std::filesystem::current_path().string());
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);

src/libcmd/include/nix/cmd/command.hh

@@ -18,7 +18,7 @@ extern char ** savedArgv;
 class EvalState;
 struct Pos;
 class Store;
-class LocalFSStore;
+struct LocalFSStore;
 
 static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
@@ -287,13 +287,16 @@ struct StorePathCommand : public StorePathsCommand
 struct RegisterCommand
 {
     typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
-    static Commands * commands;
+
+    static Commands & commands()
+    {
+        static Commands commands;
+        return commands;
+    }
 
     RegisterCommand(std::vector<std::string> && name, std::function<ref<Command>()> command)
     {
-        if (!commands)
-            commands = new Commands;
-        commands->emplace(name, command);
+        commands().emplace(name, command);
     }
 
     static nix::Commands getCommandsFor(const std::vector<std::string> & prefix);
@@ -365,7 +368,7 @@ void completeFlakeRefWithFragment(
     const Strings & defaultFlakeAttrPaths,
     std::string_view prefix);
 
-std::string showVersions(const std::set<std::string> & versions);
+std::string showVersions(const StringSet & versions);
 
 void printClosureDiff(
     ref<Store> store, const StorePath & beforePath, const StorePath & afterPath, std::string_view indent);
@@ -376,4 +379,41 @@ void printClosureDiff(
  */
 void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store);
 
-}
+/** `outLink` parameter, `createOutLinksMaybe` method. See `MixOutLinkByDefault`. */
+struct MixOutLinkBase : virtual Args
+{
+    /** Prefix for any output symlinks. Empty means do not write an output symlink. */
+    Path outLink;
+
+    MixOutLinkBase(const std::string & defaultOutLink)
+        : outLink(defaultOutLink)
+    {
+    }
+
+    void createOutLinksMaybe(const std::vector<BuiltPathWithResult> & buildables, ref<Store> & store);
+};
+
+/** `--out-link`, `--no-link`, `createOutLinksMaybe` */
+struct MixOutLinkByDefault : MixOutLinkBase, virtual Args
+{
+    MixOutLinkByDefault()
+        : MixOutLinkBase("result")
+    {
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.",
+            .labels = {"path"},
+            .handler = {&outLink},
+            .completer = completePath,
+        });
+
+        addFlag({
+            .longName = "no-link",
+            .description = "Do not create symlinks to the build results.",
+            .handler = {&outLink, Path("")},
+        });
+    }
+};
+
+} // namespace nix

src/libcmd/include/nix/cmd/installable-value.hh

@@ -21,6 +21,7 @@ struct App
 struct UnresolvedApp
 {
     App unresolved;
+    std::vector<BuiltPathWithResult> build(ref<Store> evalStore, ref<Store> store);
     App resolve(ref<Store> evalStore, ref<Store> store);
 };
 

src/libcmd/include/nix/cmd/legacy.hh

@@ -12,12 +12,15 @@ typedef std::function<void(int, char * *)> MainFunction;
 struct RegisterLegacyCommand
 {
     typedef std::map<std::string, MainFunction> Commands;
-    static Commands * commands;
+
+    static Commands & commands() {
+        static Commands commands;
+        return commands;
+    }
 
     RegisterLegacyCommand(const std::string & name, MainFunction fun)
     {
-        if (!commands) commands = new Commands;
-        (*commands)[name] = fun;
+        commands()[name] = fun;
     }
 };
 

src/libcmd/installable-attr-path.cc

@@ -72,7 +72,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
 
         auto newOutputs = std::visit(overloaded {
             [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
-                std::set<std::string> outputsToInstall;
+                StringSet outputsToInstall;
                 for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
                 if (outputsToInstall.empty())

src/libcmd/installable-flake.cc

@@ -117,7 +117,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
             .drvPath = makeConstantStorePathRef(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
-                    std::set<std::string> outputsToInstall;
+                    StringSet outputsToInstall;
                     if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) {
                         if (aOutputSpecified->getBool()) {
                             if (auto aOutputName = attr->maybeGetAttr("outputName"))

src/libcmd/installables.cc

@@ -31,8 +31,6 @@
 
 namespace nix {
 
-namespace fs { using namespace std::filesystem; }
-
 void completeFlakeInputAttrPath(
     AddCompletions & completions,
     ref<EvalState> evalState,
@@ -343,7 +341,7 @@ void completeFlakeRefWithFragment(
             auto flakeRefS = std::string(prefix.substr(0, hash));
 
             // TODO: ideally this would use the command base directory instead of assuming ".".
-            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string());
+            auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), std::filesystem::current_path().string());
 
             auto evalCache = openEvalCache(*evalState,
                 std::make_shared<flake::LockedFlake>(lockFlake(
@@ -488,7 +486,11 @@ Installables SourceExprCommand::parseInstallables(
             throw UsageError("'--file' and '--expr' are exclusive");
 
         // FIXME: backward compatibility hack
-        if (file) evalSettings.pureEval = false;
+        if (file) {
+            if (evalSettings.pureEval && evalSettings.pureEval.overridden)
+                throw UsageError("'--file' is not compatible with '--pure-eval'");
+            evalSettings.pureEval = false;
+        }
 
         auto state = getEvalState();
         auto vFile = state->allocValue();

src/libcmd/legacy.cc

@@ -1,7 +0,0 @@
-#include "nix/cmd/legacy.hh"
-
-namespace nix {
-
-RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0;
-
-}

src/libcmd/meson.build

@@ -71,7 +71,6 @@ sources = files(
   'installable-flake.cc',
   'installable-value.cc',
   'installables.cc',
-  'legacy.cc',
   'markdown.cc',
   'misc-store-flags.cc',
   'network-proxy.cc',

src/libcmd/repl-interacter.cc

@@ -135,7 +135,7 @@ static constexpr const char * promptForType(ReplPromptType promptType)
     case ReplPromptType::ReplPrompt:
         return "nix-repl> ";
     case ReplPromptType::ContinuationPrompt:
-        return "          ";
+        return "        > "; // 9 spaces + >
     }
     assert(false);
 }

src/libcmd/repl.cc

@@ -12,7 +12,7 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/attr-path.hh"
 #include "nix/util/signals.hh"
-#include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/log-store.hh"
 #include "nix/cmd/common-eval-args.hh"
 #include "nix/expr/get-drvs.hh"
@@ -61,7 +61,10 @@ struct NixRepl
 {
     size_t debugTraceIndex;
 
+    // Arguments passed to :load, saved so they can be reloaded with :reload
     Strings loadedFiles;
+    // Arguments passed to :load-flake, saved so they can be reloaded with :reload
+    Strings loadedFlakes;
     std::function<AnnotatedValues()> getValues;
 
     const static int envSize = 32768;
@@ -90,7 +93,8 @@ struct NixRepl
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
     void loadFiles();
-    void reloadFiles();
+    void loadFlakes();
+    void reloadFilesAndFlakes();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
     Expr * parseString(std::string s);
@@ -244,14 +248,13 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
         try {
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
-            for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
+            for (auto & entry : DirectoryIterator{dir == "" ? "/" : dir}) {
                 checkInterrupt();
                 auto name = entry.path().filename().string();
                 if (name[0] != '.' && hasPrefix(name, prefix2))
                     completions.insert(prev + entry.path().string());
             }
         } catch (Error &) {
-        } catch (std::filesystem::filesystem_error &) {
         }
     } else if ((dot = cur.rfind('.')) == std::string::npos) {
         /* This is a variable name; look it up in the current scope. */
@@ -467,7 +470,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
     else if (command == ":r" || command == ":reload") {
         state->resetFileCache();
-        reloadFiles();
+        reloadFilesAndFlakes();
     }
 
     else if (command == ":e" || command == ":edit") {
@@ -502,7 +505,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // Reload right after exiting the editor
         state->resetFileCache();
-        reloadFiles();
+        reloadFilesAndFlakes();
     }
 
     else if (command == ":t") {
@@ -717,6 +720,9 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
     if (flakeRefS.empty())
         throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)");
 
+    loadedFlakes.remove(flakeRefS);
+    loadedFlakes.push_back(flakeRefS);
+
     std::filesystem::path cwd;
     try {
         cwd = std::filesystem::current_path();
@@ -755,11 +761,12 @@ void NixRepl::initEnv()
 }
 
 
-void NixRepl::reloadFiles()
+void NixRepl::reloadFilesAndFlakes()
 {
     initEnv();
 
     loadFiles();
+    loadFlakes();
 }
 
 
@@ -780,6 +787,18 @@ void NixRepl::loadFiles()
 }
 
 
+void NixRepl::loadFlakes()
+{
+    Strings old = loadedFlakes;
+    loadedFlakes.clear();
+
+    for (auto & i : old) {
+        notice("Loading flake '%1%'...", i);
+        loadFlake(i);
+    }
+}
+
+
 void NixRepl::addAttrsToScope(Value & attrs)
 {
     state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope");

src/libexpr-c/meson.build

@@ -37,13 +37,11 @@ include_dirs = [include_directories('.')]
 
 headers = files(
   'nix_api_expr.h',
+  'nix_api_expr_internal.h',
   'nix_api_external.h',
   'nix_api_value.h',
 )
 
-# TODO move this header to libexpr, maybe don't use it in tests?
-headers += files('nix_api_expr_internal.h')
-
 subdir('nix-meson-build-support/export-all-symbols')
 subdir('nix-meson-build-support/windows-version')
 

src/libexpr-c/nix_api_expr.cc

@@ -199,7 +199,9 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
             != NIX_OK)
         return nullptr;
 
-    return nix_eval_state_build(context, builder);
+    auto *state = nix_eval_state_build(context, builder);
+    nix_eval_state_builder_free(builder);
+    return state;
 }
 
 void nix_state_free(EvalState * state)

src/libexpr-c/nix_api_expr.h

@@ -286,6 +286,11 @@ nix_err nix_gc_incref(nix_c_context * context, const void * object);
 /**
  * @brief Decrement the garbage collector reference counter for the given object
  *
+ * We also provide typed `nix_*_decref` functions, which are
+ *   - safer to use
+ *   - easier to integrate when deriving bindings
+ *   - allow more flexibility
+ *
  * @param[out] context Optional, stores error information
  * @param[in] object The object to stop referencing
  */

src/libexpr-tests/error_traces.cc

@@ -33,7 +33,7 @@ namespace nix {
                 ASSERT_EQ(PrintToString(e.info().msg),
                           PrintToString(HintFmt("puppy")));
                 auto trace = e.info().traces.rbegin();
-                ASSERT_EQ(e.info().traces.size(), 2);
+                ASSERT_EQ(e.info().traces.size(), 2u);
                 ASSERT_EQ(PrintToString(trace->hint),
                           PrintToString(HintFmt("doggy")));
                 trace++;
@@ -54,8 +54,8 @@ namespace nix {
             } catch (Error & e2) {
                 e.addTrace(state.positions[noPos], "beans2");
                 //e2.addTrace(state.positions[noPos], "Something", "");
-                ASSERT_TRUE(e.info().traces.size() == 2);
-                ASSERT_TRUE(e2.info().traces.size() == 0);
+                ASSERT_TRUE(e.info().traces.size() == 2u);
+                ASSERT_TRUE(e2.info().traces.size() == 0u);
                 ASSERT_FALSE(&e.info() == &e2.info());
             }
         }
@@ -71,7 +71,7 @@ namespace nix {
             } catch (BaseError & e) {                                       \
                 ASSERT_EQ(PrintToString(e.info().msg),                      \
                           PrintToString(message));                          \
-                ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \
+                ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \
                 auto trace = e.info().traces.rbegin();                      \
                 ASSERT_EQ(PrintToString(trace->hint),                       \
                           PrintToString(HintFmt("while calling the '%s' builtin", name))); \
@@ -90,7 +90,7 @@ namespace nix {
             } catch (BaseError & e) {                                       \
                 ASSERT_EQ(PrintToString(e.info().msg),                      \
                           PrintToString(message));                          \
-                ASSERT_EQ(e.info().traces.size(), 2) << "while testing " args << std::endl << e.what(); \
+                ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \
                 auto trace = e.info().traces.rbegin();                      \
                 ASSERT_EQ(PrintToString(trace->hint),                       \
                           PrintToString(context));                          \
@@ -112,7 +112,7 @@ namespace nix {
             } catch (BaseError & e) {                                       \
                 ASSERT_EQ(PrintToString(e.info().msg),                      \
                           PrintToString(message));                          \
-                ASSERT_EQ(e.info().traces.size(), 3) << "while testing " args << std::endl << e.what(); \
+                ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \
                 auto trace = e.info().traces.rbegin();                      \
                 ASSERT_EQ(PrintToString(trace->hint),                       \
                           PrintToString(context1));                         \
@@ -137,7 +137,7 @@ namespace nix {
             } catch (BaseError & e) {                                       \
                 ASSERT_EQ(PrintToString(e.info().msg),                      \
                           PrintToString(message));                          \
-                ASSERT_EQ(e.info().traces.size(), 4) << "while testing " args << std::endl << e.what(); \
+                ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \
                 auto trace = e.info().traces.rbegin();                      \
                 ASSERT_EQ(PrintToString(trace->hint),                       \
                           PrintToString(context1));                         \

src/libexpr-tests/nix_api_expr.cc

@@ -222,7 +222,7 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context)
         names.push_back(name);
     }
     std::sort(names.begin(), names.end());
-    ASSERT_EQ(3, names.size());
+    ASSERT_EQ(3u, names.size());
     EXPECT_THAT(names[0], testing::StrEq("just-a-file"));
     EXPECT_THAT(names[1], testing::StrEq("letsbuild"));
     EXPECT_THAT(names[2], testing::StrEq("not-actually-built-yet.drv"));

src/libexpr-tests/nix_api_external.cc

@@ -63,6 +63,9 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external)
     std::string string_value;
     nix_get_string(nullptr, valueResult, OBSERVE_STRING(string_value));
     ASSERT_STREQ("nix-external<MyExternalValueDesc( 42 )>", string_value.c_str());
+
+    nix_state_free(stateResult);
+    nix_state_free(stateFn);
 }
 
 }

src/libexpr-tests/nix_api_value.cc

@@ -134,12 +134,12 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list_invalid)
 {
     ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, nullptr, state, 0));
     assert_ctx_err();
-    ASSERT_EQ(0, nix_get_list_size(ctx, nullptr));
+    ASSERT_EQ(0u, nix_get_list_size(ctx, nullptr));
     assert_ctx_err();
 
     ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 0));
     assert_ctx_err();
-    ASSERT_EQ(0, nix_get_list_size(ctx, value));
+    ASSERT_EQ(0u, nix_get_list_size(ctx, value));
     assert_ctx_err();
 }
 
@@ -163,7 +163,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list)
     ASSERT_EQ(42, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 0)));
     ASSERT_EQ(43, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 1)));
     ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 2));
-    ASSERT_EQ(10, nix_get_list_size(ctx, value));
+    ASSERT_EQ(10u, nix_get_list_size(ctx, value));
 
     ASSERT_STREQ("a list", nix_get_typename(ctx, value));
     ASSERT_EQ(NIX_TYPE_LIST, nix_get_type(ctx, value));
@@ -180,7 +180,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid)
     assert_ctx_err();
     ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, nullptr, state, 0));
     assert_ctx_err();
-    ASSERT_EQ(0, nix_get_attrs_size(ctx, nullptr));
+    ASSERT_EQ(0u, nix_get_attrs_size(ctx, nullptr));
     assert_ctx_err();
     ASSERT_EQ(false, nix_has_attr_byname(ctx, nullptr, state, "no-value"));
     assert_ctx_err();
@@ -191,7 +191,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid)
     assert_ctx_err();
     ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 0));
     assert_ctx_err();
-    ASSERT_EQ(0, nix_get_attrs_size(ctx, value));
+    ASSERT_EQ(0u, nix_get_attrs_size(ctx, value));
     assert_ctx_err();
     ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value"));
     assert_ctx_err();
@@ -215,7 +215,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr)
     nix_make_attrs(ctx, value, builder);
     nix_bindings_builder_free(builder);
 
-    ASSERT_EQ(2, nix_get_attrs_size(ctx, value));
+    ASSERT_EQ(2u, nix_get_attrs_size(ctx, value));
 
     nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a");
     ASSERT_EQ(42, nix_get_int(ctx, out_value));
@@ -374,7 +374,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg)
 
     auto n = nix_get_attrs_size(ctx, r);
     assert_ctx_ok();
-    ASSERT_EQ(1, n);
+    ASSERT_EQ(1u, n);
 
     // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception
     nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo");

src/libexpr-tests/primops.cc

@@ -56,11 +56,31 @@ namespace nix {
     TEST_F(PrimOpTest, ceil) {
         auto v = eval("builtins.ceil 1.9");
         ASSERT_THAT(v, IsIntEq(2));
+        auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)");
+        ASSERT_THAT(intMin, IsIntEq(std::numeric_limits<NixInt::Inner>::min()));
+        ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf
+        ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf
+        ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan
+        // bugs in previous Nix versions
+        ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError);
     }
 
     TEST_F(PrimOpTest, floor) {
         auto v = eval("builtins.floor 1.9");
         ASSERT_THAT(v, IsIntEq(1));
+        auto intMin = eval("builtins.ceil (-4611686018427387904 - 4611686018427387904)");
+        ASSERT_THAT(intMin, IsIntEq(std::numeric_limits<NixInt::Inner>::min()));
+        ASSERT_THROW(eval("builtins.ceil 1.0e200"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil -1.0e200"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200)"), EvalError); // inf
+        ASSERT_THROW(eval("builtins.ceil (-1.0e200 * 1.0e200)"), EvalError); // -inf
+        ASSERT_THROW(eval("builtins.ceil (1.0e200 * 1.0e200 - 1.0e200 * 1.0e200)"), EvalError); // nan
+        // bugs in previous Nix versions
+        ASSERT_THROW(eval("builtins.ceil (4611686018427387904 + 4611686018427387903)"), EvalError);
+        ASSERT_THROW(eval("builtins.ceil (-4611686018427387904 - 4611686018427387903)"), EvalError);
     }
 
     TEST_F(PrimOpTest, tryEvalFailure) {
@@ -204,7 +224,7 @@ namespace nix {
         auto v = eval("builtins.listToAttrs []");
         ASSERT_THAT(v, IsAttrsOfSize(0));
         ASSERT_EQ(v.type(), nAttrs);
-        ASSERT_EQ(v.attrs()->size(), 0);
+        ASSERT_EQ(v.attrs()->size(), 0u);
     }
 
     TEST_F(PrimOpTest, listToAttrsNotFieldName) {
@@ -383,7 +403,7 @@ namespace nix {
     TEST_F(PrimOpTest, genList) {
         auto v = eval("builtins.genList (x: x + 1) 3");
         ASSERT_EQ(v.type(), nList);
-        ASSERT_EQ(v.listSize(), 3);
+        ASSERT_EQ(v.listSize(), 3u);
         for (const auto [i, elem] : enumerate(v.listItems())) {
             ASSERT_THAT(*elem, IsThunk());
             state.forceValue(*elem, noPos);
@@ -394,7 +414,7 @@ namespace nix {
     TEST_F(PrimOpTest, sortLessThan) {
         auto v = eval("builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ]");
         ASSERT_EQ(v.type(), nList);
-        ASSERT_EQ(v.listSize(), 6);
+        ASSERT_EQ(v.listSize(), 6u);
 
         const std::vector<int> numbers = { 42, 77, 147, 249, 483, 526 };
         for (const auto [n, elem] : enumerate(v.listItems()))
@@ -414,7 +434,7 @@ namespace nix {
         auto wrong = v.attrs()->get(createSymbol("wrong"));
         ASSERT_NE(wrong, nullptr);
         ASSERT_EQ(wrong->value->type(), nList);
-        ASSERT_EQ(wrong->value->listSize(), 3);
+        ASSERT_EQ(wrong->value->listSize(), 3u);
         ASSERT_THAT(*wrong->value, IsListOfSize(3));
         ASSERT_THAT(*wrong->value->listElems()[0], IsIntEq(1));
         ASSERT_THAT(*wrong->value->listElems()[1], IsIntEq(9));
@@ -424,7 +444,7 @@ namespace nix {
     TEST_F(PrimOpTest, concatMap) {
         auto v = eval("builtins.concatMap (x: x ++ [0]) [ [1 2] [3 4] ]");
         ASSERT_EQ(v.type(), nList);
-        ASSERT_EQ(v.listSize(), 6);
+        ASSERT_EQ(v.listSize(), 6u);
 
         const std::vector<int> numbers = { 1, 2, 0, 3, 4, 0 };
         for (const auto [n, elem] : enumerate(v.listItems()))

src/libexpr/attr-path.cc

@@ -74,7 +74,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin
 
             auto a = v->attrs()->get(state.symbols.create(attr));
             if (!a) {
-                std::set<std::string> attrNames;
+                StringSet attrNames;
                 for (auto & attr : *v->attrs())
                     attrNames.insert(std::string(state.symbols[attr.name]));
 

src/libexpr/eval-cache.cc

@@ -492,7 +492,7 @@ Value & AttrCursor::forceValue()
 Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
 {
     auto attrNames = getAttrs();
-    std::set<std::string> strAttrNames;
+    StringSet strAttrNames;
     for (auto & name : attrNames)
         strAttrNames.insert(std::string(root->state.symbols[name]));
 

src/libexpr/eval-error.cc

@@ -110,5 +110,6 @@ template class EvalErrorBuilder<UndefinedVarError>;
 template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
 template class EvalErrorBuilder<InvalidPathError>;
+template class EvalErrorBuilder<IFDError>;
 
 }

src/libexpr/eval.cc

@@ -252,7 +252,25 @@ EvalState::EvalState(
         makeMountedSourceAccessor(
             {
                 {CanonPath::root, makeEmptySourceAccessor()},
-                {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))}
+                /* In the pure eval case, we can simply require
+                   valid paths. However, in the *impure* eval
+                   case this gets in the way of the union
+                   mechanism, because an invalid access in the
+                   upper layer will *not* be caught by the union
+                   source accessor, but instead abort the entire
+                   lookup.
+
+                   This happens when the store dir in the
+                   ambient file system has a path (e.g. because
+                   another Nix store there), but the relocated
+                   store does not.
+
+                   TODO make the various source accessors doing
+                   access control all throw the same type of
+                   exception, and make union source accessor
+                   catch it, so we don't need to do this hack.
+                 */
+                {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)},
             }))
     , rootFS(
         ({
@@ -1435,7 +1453,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             } else {
                 state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
                 if (!(j = vAttrs->attrs()->get(name))) {
-                    std::set<std::string> allAttrNames;
+                    StringSet allAttrNames;
                     for (auto & attr : *vAttrs->attrs())
                         allAttrNames.insert(std::string(state.symbols[attr.name]));
                     auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
@@ -1592,7 +1610,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
                        user. */
                     for (auto & i : *args[0]->attrs())
                         if (!lambda.formals->has(i.name)) {
-                            std::set<std::string> formalNames;
+                            StringSet formalNames;
                             for (auto & formal : lambda.formals->formals)
                                 formalNames.insert(std::string(symbols[formal.name]));
                             auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);

src/libexpr/include/nix/expr/eval-error.hh

@@ -54,6 +54,7 @@ MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, EvalError);
 MakeError(MissingArgumentError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
+MakeError(IFDError, EvalBaseError);
 
 struct InvalidPathError : public EvalError
 {

src/libexpr/include/nix/expr/primops.hh

@@ -27,7 +27,12 @@ constexpr size_t conservativeStackReservation = 16;
 struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;
-    static PrimOps * primOps;
+
+    static PrimOps & primOps()
+    {
+        static PrimOps primOps;
+        return primOps;
+    }
 
     /**
      * You can register a constant by passing an arity of 0. fun

src/libexpr/include/nix/expr/symbol-table.hh

@@ -81,26 +81,29 @@ public:
 class SymbolTable
 {
 private:
-    std::unordered_map<std::string_view, std::pair<const std::string *, uint32_t>> symbols;
+    /**
+     * Map from string view (backed by ChunkedVector) -> offset into the store.
+     * ChunkedVector references are never invalidated.
+     */
+    std::unordered_map<std::string_view, uint32_t> symbols;
     ChunkedVector<std::string, 8192> store{16};
 
 public:
 
     /**
-     * converts a string into a symbol.
+     * Converts a string into a symbol.
      */
     Symbol create(std::string_view s)
     {
         // Most symbols are looked up more than once, so we trade off insertion performance
         // for lookup performance.
-        // TODO: could probably be done more efficiently with transparent Hash and Equals
-        // on the original implementation using unordered_set
         // FIXME: make this thread-safe.
         auto it = symbols.find(s);
-        if (it != symbols.end()) return Symbol(it->second.second + 1);
+        if (it != symbols.end())
+            return Symbol(it->second + 1);
 
-        const auto & [rawSym, idx] = store.add(std::string(s));
-        symbols.emplace(rawSym, std::make_pair(&rawSym, idx));
+        const auto & [rawSym, idx] = store.add(s);
+        symbols.emplace(rawSym, idx);
         return Symbol(idx + 1);
     }
 

src/libexpr/lexer.l

@@ -1,3 +1,4 @@
+%option 8bit
 %option reentrant bison-bridge bison-locations
 %option align
 %option noyywrap

src/libexpr/meson.build

@@ -112,6 +112,7 @@ lexer_tab = custom_target(
   ],
   command : [
     'flex',
+    '-Cf', # Use full scanner tables
     '--outfile',
     '@OUTPUT0@',
     '--header-file=' + '@OUTPUT1@',

src/libexpr/parser.y

@@ -179,7 +179,12 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 
 %%
 
-start: expr { state->result = $1; };
+start: expr {
+  state->result = $1;
+
+  // This parser does not use yynerrs; suppress the warning.
+  (void) yynerrs;
+};
 
 expr: expr_function;
 

src/libexpr/primops.cc

@@ -98,7 +98,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
     if (drvs.empty()) return {};
 
     if (isIFD && !settings.enableImportFromDerivation)
-        error<EvalBaseError>(
+        error<IFDError>(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             drvs.begin()->to_string(*store)
         ).debugThrow();
@@ -895,18 +895,40 @@ static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value
 {
     auto value = state.forceFloat(*args[0], args[0]->determinePos(pos),
             "while evaluating the first argument passed to builtins.ceil");
-    v.mkInt(ceil(value));
+    auto ceilValue = ceil(value);
+    bool isInt = args[0]->type() == nInt;
+    constexpr NixFloat int_min = std::numeric_limits<NixInt::Inner>::min(); // power of 2, so that no rounding occurs
+    if (ceilValue >= int_min && ceilValue < -int_min) {
+        v.mkInt(ceilValue);
+    } else if (isInt) {
+        // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat
+        state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow();
+    } else {
+        state.error<EvalError>("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow();
+    }
+    // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified
+    if (isInt) {
+        auto arg = args[0]->integer();
+        auto res = v.integer();
+        if (arg != res) {
+            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_ceil({
     .name = "__ceil",
-    .args = {"double"},
+    .args = {"number"},
     .doc = R"(
-        Converts an IEEE-754 double-precision floating-point number (*double*) to
-        the next higher integer.
+        Rounds and converts *number* to the next higher NixInt value if possible, i.e. `ceil *number* >= *number*` and
+        `ceil *number* - *number* < 1`.
 
-        If the datatype is neither an integer nor a "float", an evaluation error will be
-        thrown.
+        An evaluation error is thrown, if there exists no such NixInt value `ceil *number*`.
+        Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is
+        a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`.
+
+        If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat
+        (IEEE-754 double-precision floating-point number), an evaluation error will be thrown.
     )",
     .fun = prim_ceil,
 });
@@ -914,18 +936,40 @@ static RegisterPrimOp primop_ceil({
 static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor");
-    v.mkInt(floor(value));
+    auto floorValue = floor(value);
+    bool isInt = args[0]->type() == nInt;
+    constexpr NixFloat int_min = std::numeric_limits<NixInt::Inner>::min(); // power of 2, so that no rounding occurs
+    if (floorValue >= int_min && floorValue < -int_min) {
+        v.mkInt(floorValue);
+    } else if (isInt) {
+        // a NixInt, e.g. INT64_MAX, can be rounded to -int_min due to the cast to NixFloat
+        state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) the NixInt argument %1% caused undefined behavior in previous Nix versions.\n\tFuture Nix versions might implement the correct behavior.", args[0]->integer().value).atPos(pos).debugThrow();
+    } else {
+        state.error<EvalError>("NixFloat argument %1% is not in the range of NixInt", args[0]->fpoint()).atPos(pos).debugThrow();
+    }
+    // `forceFloat` casts NixInt to NixFloat, but instead NixInt args shall be returned unmodified
+    if (isInt) {
+        auto arg = args[0]->integer();
+        auto res = v.integer();
+        if (arg != res) {
+            state.error<EvalError>("Due to a bug (see https://github.com/NixOS/nix/issues/12899) a loss of precision occured in previous Nix versions because the NixInt argument %1% was rounded to %2%.\n\tFuture Nix versions might implement the correct behavior.", arg, res).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_floor({
     .name = "__floor",
-    .args = {"double"},
+    .args = {"number"},
     .doc = R"(
-        Converts an IEEE-754 double-precision floating-point number (*double*) to
-        the next lower integer.
+        Rounds and converts *number* to the next lower NixInt value if possible, i.e. `floor *number* <= *number*` and
+        `*number* - floor *number* < 1`.
 
-        If the datatype is neither an integer nor a "float", an evaluation error will be
-        thrown.
+        An evaluation error is thrown, if there exists no such NixInt value `floor *number*`.
+        Due to bugs in previous Nix versions an evaluation error might be thrown, if the datatype of *number* is
+        a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`.
+
+        If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat
+        (IEEE-754 double-precision floating-point number), an evaluation error will be thrown.
     )",
     .fun = prim_floor,
 });
@@ -2813,7 +2857,13 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * *
 
 static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
     .name = "__unsafeGetAttrPos",
+    .args = {"s", "set"},
     .arity = 2,
+    .doc = R"(
+      `unsafeGetAttrPos` returns the position of the attribute named *s*
+      from *set*. This is used by Nixpkgs to provide location information
+      in error messages.
+    )",
     .fun = prim_unsafeGetAttrPos,
 });
 
@@ -4299,9 +4349,7 @@ struct RegexCache
 {
     struct State
     {
-        // TODO use C++20 transparent comparison when available
-        std::unordered_map<std::string_view, std::regex> cache;
-        std::list<std::string> keys;
+        std::unordered_map<std::string, std::regex, StringViewHash, std::equal_to<>> cache;
     };
 
     Sync<State> state_;
@@ -4312,8 +4360,14 @@ struct RegexCache
         auto it = state->cache.find(re);
         if (it != state->cache.end())
             return it->second;
-        state->keys.emplace_back(re);
-        return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second;
+        /* No std::regex constructor overload from std::string_view, but can be constructed
+           from a pointer + size or an iterator range. */
+        return state->cache
+            .emplace(
+                std::piecewise_construct,
+                std::forward_as_tuple(re),
+                std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended))
+            .first->second;
     }
 };
 
@@ -4697,13 +4751,9 @@ static RegisterPrimOp primop_splitVersion({
  *************************************************************/
 
 
-RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
-
-
 RegisterPrimOp::RegisterPrimOp(PrimOp && primOp)
 {
-    if (!primOps) primOps = new PrimOps;
-    primOps->push_back(std::move(primOp));
+    primOps().push_back(std::move(primOp));
 }
 
 
@@ -4957,14 +5007,18 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         )",
     });
 
-    if (RegisterPrimOp::primOps)
-        for (auto & primOp : *RegisterPrimOp::primOps)
-            if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature))
-            {
-                auto primOpAdjusted = primOp;
-                primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity);
-                addPrimOp(std::move(primOpAdjusted));
-            }
+    for (auto & primOp : RegisterPrimOp::primOps())
+        if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) {
+            auto primOpAdjusted = primOp;
+            primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity);
+            addPrimOp(std::move(primOpAdjusted));
+        }
+
+    for (auto & primOp : evalSettings.extraPrimOps) {
+        auto primOpAdjusted = primOp;
+        primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity);
+        addPrimOp(std::move(primOpAdjusted));
+    }
 
     for (auto & primOp : evalSettings.extraPrimOps) {
         auto primOpAdjusted = primOp;

src/libexpr/primops/fetchClosure.cc

@@ -1,5 +1,5 @@
 #include "nix/expr/primops.hh"
-#include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/realisation.hh"
 #include "nix/store/make-content-addressed.hh"
 #include "nix/util/url.hh"

src/libexpr/value-to-json.cc

@@ -11,6 +11,7 @@ namespace nix {
 
 using json = nlohmann::json;
 
+// TODO: rename. It doesn't print.
 json printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore)
 {

src/libfetchers-c/.version

@@ -0,0 +1 @@
+../../.version

src/libfetchers-c/meson.build

@@ -0,0 +1,65 @@
+project('nix-fetchers-c', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('nix-meson-build-support/deps-lists')
+
+deps_private_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+  dependency('nix-fetchers'),
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util-c'),
+  dependency('nix-store-c'),
+]
+subdir('nix-meson-build-support/subprojects')
+
+add_project_arguments(
+  language : 'cpp',
+)
+
+subdir('nix-meson-build-support/common')
+
+sources = files(
+  'nix_api_fetchers.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = files(
+  'nix_api_fetchers.h',
+  'nix_api_fetchers_internal.hh',
+)
+
+# TODO move this header to libexpr, maybe don't use it in tests?
+headers += files('nix_api_fetchers.h')
+
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
+this_library = library(
+  'nixfetchersc',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, preserve_path : true)
+
+libraries_private = []
+
+subdir('nix-meson-build-support/export')

src/libfetchers-c/nix-meson-build-support

@@ -0,0 +1 @@
+../../nix-meson-build-support

src/libfetchers-c/nix_api_fetchers.cc

@@ -0,0 +1,19 @@
+#include "nix_api_fetchers.h"
+#include "nix_api_fetchers_internal.hh"
+#include "nix_api_util_internal.h"
+
+nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context)
+{
+    try {
+        auto fetchersSettings = nix::make_ref<nix::fetchers::Settings>(nix::fetchers::Settings{});
+        return new nix_fetchers_settings{
+            .settings = fetchersSettings,
+        };
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_fetchers_settings_free(nix_fetchers_settings * settings)
+{
+    delete settings;
+}

src/libfetchers-c/nix_api_fetchers.h

@@ -0,0 +1,32 @@
+#ifndef NIX_API_FETCHERS_H
+#define NIX_API_FETCHERS_H
+/** @defgroup libfetchers libfetchers
+ * @brief Bindings to the Nix fetchers library
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libfetchers C bindings
+ */
+
+#include "nix_api_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Shared settings object
+ */
+typedef struct nix_fetchers_settings nix_fetchers_settings;
+
+nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context);
+
+void nix_fetchers_settings_free(nix_fetchers_settings * settings);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // NIX_API_FETCHERS_H

src/libfetchers-c/nix_api_fetchers_internal.hh

@@ -0,0 +1,12 @@
+#pragma once
+#include "nix/fetchers/fetch-settings.hh"
+#include "nix/util/ref.hh"
+
+/**
+ * A shared reference to `nix::fetchers::Settings`
+ * @see nix::fetchers::Settings
+ */
+struct nix_fetchers_settings
+{
+    nix::ref<nix::fetchers::Settings> settings;
+};

src/libfetchers-c/package.nix

@@ -0,0 +1,50 @@
+{
+  lib,
+  mkMesonLibrary,
+
+  nix-store-c,
+  nix-expr-c,
+  nix-util-c,
+  nix-fetchers,
+
+  # Configuration Options
+
+  version,
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonLibrary (finalAttrs: {
+  pname = "nix-fetchers-c";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../nix-meson-build-support
+    ./nix-meson-build-support
+    ../../.version
+    ./.version
+    ./meson.build
+    # ./meson.options
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+    (fileset.fileFilter (file: file.hasExt "h") ./.)
+  ];
+
+  propagatedBuildInputs = [
+    nix-util-c
+    nix-expr-c
+    nix-store-c
+    nix-fetchers
+  ];
+
+  mesonFlags = [
+  ];
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libfetchers-tests/git-utils.cc

@@ -11,14 +11,10 @@
 
 namespace nix {
 
-namespace fs {
-using namespace std::filesystem;
-}
-
 class GitUtilsTest : public ::testing::Test
 {
     // We use a single repository for all tests.
-    fs::path tmpDir;
+    std::filesystem::path tmpDir;
     std::unique_ptr<AutoDelete> delTmpDir;
 
 public:
@@ -90,11 +86,11 @@ TEST_F(GitUtilsTest, sink_basic)
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
     auto accessor = repo->getAccessor(result, false, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
-    ASSERT_EQ(entries.size(), 5);
+    ASSERT_EQ(entries.size(), 5u);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
     ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish");
     ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye");
-    ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0);
+    ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0u);
     ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world");
 };
 

src/libfetchers-tests/meson.build

@@ -17,6 +17,7 @@ subdir('nix-meson-build-support/deps-lists')
 deps_private_maybe_subproject = [
   dependency('nix-store-test-support'),
   dependency('nix-fetchers'),
+  dependency('nix-fetchers-c'),
 ]
 deps_public_maybe_subproject = [
 ]
@@ -39,6 +40,7 @@ subdir('nix-meson-build-support/common')
 sources = files(
   'access-tokens.cc',
   'git-utils.cc',
+  'nix_api_fetchers.cc',
   'public-key.cc',
 )
 

src/libfetchers-tests/nix_api_fetchers.cc

@@ -0,0 +1,18 @@
+#include "gmock/gmock.h"
+#include <gtest/gtest.h>
+
+#include "nix_api_fetchers.h"
+#include "nix/store/tests/nix_api_store.hh"
+
+namespace nixC {
+
+TEST_F(nix_api_store_test, nix_api_fetchers_new_free)
+{
+    nix_fetchers_settings * settings = nix_fetchers_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, settings);
+
+    nix_fetchers_settings_free(settings);
+}
+
+} // namespace nixC

src/libfetchers-tests/package.nix

@@ -5,6 +5,7 @@
   mkMesonExecutable,
 
   nix-fetchers,
+  nix-fetchers-c,
   nix-store-test-support,
 
   libgit2,
@@ -40,6 +41,7 @@ mkMesonExecutable (finalAttrs: {
 
   buildInputs = [
     nix-fetchers
+    nix-fetchers-c
     nix-store-test-support
     rapidcheck
     gtest

src/libfetchers/fetchers.cc

@@ -12,24 +12,26 @@ namespace nix::fetchers {
 
 using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
 
-std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
+static InputSchemeMap & inputSchemes()
+{
+    static InputSchemeMap inputSchemeMap;
+    return inputSchemeMap;
+}
 
 void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
 {
-    if (!inputSchemes)
-        inputSchemes = std::make_unique<InputSchemeMap>();
     auto schemeName = inputScheme->schemeName();
-    if (inputSchemes->count(schemeName) > 0)
+    if (!inputSchemes().emplace(schemeName, std::move(inputScheme)).second)
         throw Error("Input scheme with name %s already registered", schemeName);
-    inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
 }
 
-nlohmann::json dumpRegisterInputSchemeInfo() {
+nlohmann::json dumpRegisterInputSchemeInfo()
+{
     using nlohmann::json;
 
     auto res = json::object();
 
-    for (auto & [name, scheme] : *inputSchemes) {
+    for (auto & [name, scheme] : inputSchemes()) {
         auto & r = res[name] = json::object();
         r["allowedAttrs"] = scheme->allowedAttrs();
     }
@@ -57,7 +59,7 @@ Input Input::fromURL(
     const Settings & settings,
     const ParsedURL & url, bool requireTree)
 {
-    for (auto & [_, inputScheme] : *inputSchemes) {
+    for (auto & [_, inputScheme] : inputSchemes()) {
         auto res = inputScheme->inputFromURL(settings, url, requireTree);
         if (res) {
             experimentalFeatureSettings.require(inputScheme->experimentalFeature());
@@ -91,8 +93,8 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     };
 
     std::shared_ptr<InputScheme> inputScheme = ({
-        auto i = inputSchemes->find(schemeName);
-        i == inputSchemes->end() ? nullptr : i->second;
+        auto i = get(inputSchemes(), schemeName);
+        i ? *i : nullptr;
     });
 
     if (!inputScheme) return raw();

src/libfetchers/git-lfs-fetch.cc

@@ -44,10 +44,11 @@ static void downloadToSink(
 
 static std::string getLfsApiToken(const ParsedURL & url)
 {
-    auto [status, output] = runProgram(RunOptions{
-        .program = "ssh",
-        .args = {*url.authority, "git-lfs-authenticate", url.path, "download"},
-    });
+    auto [status, output] = runProgram(
+        RunOptions{
+            .program = "ssh",
+            .args = {*url.authority, "git-lfs-authenticate", url.path, "download"},
+        });
 
     if (output.empty())
         throw Error(

src/libfetchers/git-utils.cc

@@ -1244,7 +1244,12 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     auto configS = accessor->readFile(modulesFile);
 
     auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
-    writeFull(fdTemp.get(), configS);
+    try {
+        writeFull(fdTemp.get(), configS);
+    } catch (SysError & e) {
+        e.addTrace({}, "while writing .gitmodules file to temporary file");
+        throw;
+    }
 
     std::vector<std::tuple<Submodule, Hash>> result;
 

src/libfetchers/git.cc

@@ -39,7 +39,7 @@ const std::string gitInitialBranch = "__nix_dummy_branch";
 
 bool isCacheFileWithinTtl(time_t now, const struct stat & st)
 {
-    return st.st_mtime + settings.tarballTtl > now;
+    return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
 }
 
 Path getCachePath(std::string_view key, bool shallow)
@@ -84,10 +84,9 @@ std::optional<std::string> readHead(const Path & path)
 }
 
 // Persist the HEAD ref from the remote repo in the local cached repo.
-bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
+bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::string & headRef)
 {
-    // set shallow=false as HEAD will never be queried for a shallow repo
-    Path cacheDir = getCachePath(actualUrl, false);
+    Path cacheDir = getCachePath(actualUrl, shallow);
     try {
         runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
     } catch (ExecError &e) {
@@ -106,12 +105,11 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
     return true;
 }
 
-std::optional<std::string> readHeadCached(const std::string & actualUrl)
+std::optional<std::string> readHeadCached(const std::string & actualUrl, bool shallow)
 {
     // Create a cache path to store the branch of the HEAD ref. Append something
     // in front of the URL to prevent collision with the repository itself.
-    // set shallow=false as HEAD will never be queried for a shallow repo
-    Path cacheDir = getCachePath(actualUrl, false);
+    Path cacheDir = getCachePath(actualUrl, shallow);
     Path headRefFile = cacheDir + "/HEAD";
 
     time_t now = time(0);
@@ -517,14 +515,14 @@ struct GitInputScheme : InputScheme
         return revCount;
     }
 
-    std::string getDefaultRef(const RepoInfo & repoInfo) const
+    std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const
     {
         auto head = std::visit(
             overloaded {
                 [&](const std::filesystem::path & path)
                 { return GitRepo::openRepo(path)->getWorkdirRef(); },
                 [&](const ParsedURL & url)
-                { return readHeadCached(url.to_string()); }
+                { return readHeadCached(url.to_string(), shallow); }
             }, repoInfo.location);
         if (!head) {
             warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg());
@@ -536,7 +534,7 @@ struct GitInputScheme : InputScheme
     static MakeNotAllowedError makeNotAllowedError(std::filesystem::path repoPath)
     {
         return [repoPath{std::move(repoPath)}](const CanonPath & path) -> RestrictedPathError {
-            if (fs::symlink_exists(repoPath / path.rel()))
+            if (pathExists(repoPath / path.rel()))
                 return RestrictedPathError(
                     "Path '%1%' in the repository %2% is not tracked by Git.\n"
                     "\n"
@@ -573,7 +571,8 @@ struct GitInputScheme : InputScheme
         auto origRev = input.getRev();
 
         auto originalRef = input.getRef();
-        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
+        bool shallow = getShallowAttr(input);
+        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
         input.attrs.insert_or_assign("ref", ref);
 
         std::filesystem::path repoDir;
@@ -584,7 +583,7 @@ struct GitInputScheme : InputScheme
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
         } else {
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
-            std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input));
+            std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
@@ -621,6 +620,7 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
+                bool shallow = getShallowAttr(input);
                 try {
                     auto fetchRef =
                         getAllRefsAttr(input)
@@ -633,7 +633,7 @@ struct GitInputScheme : InputScheme
                         ? ref
                         : fmt("%1%:%1%", "refs/heads/" + ref);
 
-                    repo->fetch(repoUrl.to_string(), fetchRef, getShallowAttr(input));
+                    repo->fetch(repoUrl.to_string(), fetchRef, shallow);
                 } catch (Error & e) {
                     if (!std::filesystem::exists(localRefFile)) throw;
                     logError(e.info());
@@ -644,9 +644,9 @@ struct GitInputScheme : InputScheme
                     if (!input.getRev())
                         setWriteTime(localRefFile, now, now);
                 } catch (Error & e) {
-                    warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg);
+                    warn("could not update mtime for file %s: %s", localRefFile, e.info().msg);
                 }
-                if (!originalRef && !storeCachedHead(repoUrl.to_string(), ref))
+                if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref))
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
             }
 

src/libfetchers/github.cc

@@ -149,6 +149,9 @@ struct GitArchiveInputScheme : InputScheme
         };
         if (auto narHash = input.getNarHash())
             url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
+        auto host = maybeGetStrAttr(input.attrs, "host");
+        if (host)
+            url.query.insert_or_assign("host", *host);
         return url;
     }
 

src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh

@@ -4,8 +4,6 @@
 
 #include <unordered_set>
 
-#include <unordered_set>
-
 namespace nix {
 
 /**

src/libfetchers/input-cache.cc

@@ -16,7 +16,7 @@ InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegist
             auto [accessor, lockedInput] = originalInput.getAccessor(store);
             fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
         } else {
-            if (useRegistries != fetchers::UseRegistries::No) {
+            if (useRegistries != UseRegistries::No) {
                 auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries);
                 resolvedInput = std::move(res);
                 fetched = lookup(resolvedInput);

src/libfetchers/mercurial.cc

@@ -194,7 +194,7 @@ struct MercurialInputScheme : InputScheme
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
 
-                auto files = tokenizeString<std::set<std::string>>(
+                auto files = tokenizeString<StringSet>(
                     runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
 
                 Path actualPath(absPath(actualUrl));

src/libfetchers/store-path-accessor.cc

@@ -5,11 +5,7 @@ namespace nix {
 
 ref<SourceAccessor> makeStorePathAccessor(ref<Store> store, const StorePath & storePath)
 {
-    // FIXME: should use `store->getFSAccessor()`
-    auto root = std::filesystem::path{store->toRealPath(storePath)};
-    auto accessor = makeFSSourceAccessor(root);
-    accessor->setPathDisplay(root.string());
-    return accessor;
+    return projectSubdirSourceAccessor(store->getFSAccessor(), storePath.to_string());
 }
 
 }

src/libfetchers/tarball.cc

@@ -224,7 +224,7 @@ ref<SourceAccessor> downloadTarball(
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
-    const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
+    const StringSet transportUrlSchemes = {"file", "http", "https"};
 
     bool hasTarballExtension(std::string_view path) const
     {
@@ -236,7 +236,7 @@ struct CurlInputScheme : InputScheme
 
     virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0;
 
-    static const std::set<std::string> specialParams;
+    static const StringSet specialParams;
 
     std::optional<Input> inputFromURL(
         const Settings & settings,

src/libflake-c/meson.build

@@ -17,12 +17,14 @@ subdir('nix-meson-build-support/deps-lists')
 deps_private_maybe_subproject = [
   dependency('nix-util'),
   dependency('nix-store'),
+  dependency('nix-fetchers'),
   dependency('nix-expr'),
   dependency('nix-flake'),
 ]
 deps_public_maybe_subproject = [
   dependency('nix-util-c'),
   dependency('nix-store-c'),
+  dependency('nix-fetchers-c'),
   dependency('nix-expr-c'),
 ]
 subdir('nix-meson-build-support/subprojects')
@@ -37,6 +39,7 @@ include_dirs = [include_directories('.')]
 
 headers = files(
   'nix_api_flake.h',
+  'nix_api_flake_internal.hh',
 )
 
 # TODO move this header to libexpr, maybe don't use it in tests?

src/libflake-c/nix_api_flake.cc

@@ -1,12 +1,18 @@
+#include <string>
+
 #include "nix_api_flake.h"
 #include "nix_api_flake_internal.hh"
+#include "nix_api_util.h"
 #include "nix_api_util_internal.h"
 #include "nix_api_expr_internal.h"
+#include "nix_api_fetchers_internal.hh"
+#include "nix_api_fetchers.h"
 
 #include "nix/flake/flake.hh"
 
 nix_flake_settings * nix_flake_settings_new(nix_c_context * context)
 {
+    nix_clear_err(context);
     try {
         auto settings = nix::make_ref<nix::flake::Settings>();
         return new nix_flake_settings{settings};
@@ -22,8 +28,178 @@ void nix_flake_settings_free(nix_flake_settings * settings)
 nix_err nix_flake_settings_add_to_eval_state_builder(
     nix_c_context * context, nix_flake_settings * settings, nix_eval_state_builder * builder)
 {
+    nix_clear_err(context);
     try {
         settings->settings->configureEvalSettings(builder->settings);
     }
     NIXC_CATCH_ERRS
 }
+
+nix_flake_reference_parse_flags *
+nix_flake_reference_parse_flags_new(nix_c_context * context, nix_flake_settings * settings)
+{
+    nix_clear_err(context);
+    try {
+        return new nix_flake_reference_parse_flags{
+            .baseDirectory = std::nullopt,
+        };
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_flake_reference_parse_flags_free(nix_flake_reference_parse_flags * flags)
+{
+    delete flags;
+}
+
+nix_err nix_flake_reference_parse_flags_set_base_directory(
+    nix_c_context * context,
+    nix_flake_reference_parse_flags * flags,
+    const char * baseDirectory,
+    size_t baseDirectoryLen)
+{
+    nix_clear_err(context);
+    try {
+        flags->baseDirectory.emplace(nix::Path{std::string(baseDirectory, baseDirectoryLen)});
+        return NIX_OK;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_flake_reference_and_fragment_from_string(
+    nix_c_context * context,
+    nix_fetchers_settings * fetchSettings,
+    nix_flake_settings * flakeSettings,
+    nix_flake_reference_parse_flags * parseFlags,
+    const char * strData,
+    size_t strSize,
+    nix_flake_reference ** flakeReferenceOut,
+    nix_get_string_callback fragmentCallback,
+    void * fragmentCallbackUserData)
+{
+    nix_clear_err(context);
+    *flakeReferenceOut = nullptr;
+    try {
+        std::string str(strData, strSize);
+
+        auto [flakeRef, fragment] =
+            nix::parseFlakeRefWithFragment(*fetchSettings->settings, str, parseFlags->baseDirectory, true);
+        *flakeReferenceOut = new nix_flake_reference{nix::make_ref<nix::FlakeRef>(flakeRef)};
+        return call_nix_get_string_callback(fragment, fragmentCallback, fragmentCallbackUserData);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_flake_reference_free(nix_flake_reference * flakeReference)
+{
+    delete flakeReference;
+}
+
+nix_flake_lock_flags * nix_flake_lock_flags_new(nix_c_context * context, nix_flake_settings * settings)
+{
+    nix_clear_err(context);
+    try {
+        auto lockSettings = nix::make_ref<nix::flake::LockFlags>(nix::flake::LockFlags{
+            .recreateLockFile = false,
+            .updateLockFile = true,  // == `nix_flake_lock_flags_set_mode_write_as_needed`
+            .writeLockFile = true,   // == `nix_flake_lock_flags_set_mode_write_as_needed`
+            .failOnUnlocked = false, // == `nix_flake_lock_flags_set_mode_write_as_needed`
+            .useRegistries = false,
+            .allowUnlocked = false, // == `nix_flake_lock_flags_set_mode_write_as_needed`
+            .commitLockFile = false,
+
+        });
+        return new nix_flake_lock_flags{lockSettings};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_flake_lock_flags_free(nix_flake_lock_flags * flags)
+{
+    delete flags;
+}
+
+nix_err nix_flake_lock_flags_set_mode_virtual(nix_c_context * context, nix_flake_lock_flags * flags)
+{
+    nix_clear_err(context);
+    try {
+        flags->lockFlags->updateLockFile = true;
+        flags->lockFlags->writeLockFile = false;
+        flags->lockFlags->failOnUnlocked = false;
+        flags->lockFlags->allowUnlocked = true;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, nix_flake_lock_flags * flags)
+{
+    nix_clear_err(context);
+    try {
+        flags->lockFlags->updateLockFile = true;
+        flags->lockFlags->writeLockFile = true;
+        flags->lockFlags->failOnUnlocked = false;
+        flags->lockFlags->allowUnlocked = true;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_flake_lock_flags_set_mode_check(nix_c_context * context, nix_flake_lock_flags * flags)
+{
+    nix_clear_err(context);
+    try {
+        flags->lockFlags->updateLockFile = false;
+        flags->lockFlags->writeLockFile = false;
+        flags->lockFlags->failOnUnlocked = true;
+        flags->lockFlags->allowUnlocked = false;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_flake_lock_flags_add_input_override(
+    nix_c_context * context, nix_flake_lock_flags * flags, const char * inputPath, nix_flake_reference * flakeRef)
+{
+    nix_clear_err(context);
+    try {
+        auto path = nix::flake::parseInputAttrPath(inputPath);
+        flags->lockFlags->inputOverrides.emplace(path, *flakeRef->flakeRef);
+        if (flags->lockFlags->writeLockFile) {
+            return nix_flake_lock_flags_set_mode_virtual(context, flags);
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_locked_flake * nix_flake_lock(
+    nix_c_context * context,
+    nix_fetchers_settings * fetchSettings,
+    nix_flake_settings * flakeSettings,
+    EvalState * eval_state,
+    nix_flake_lock_flags * flags,
+    nix_flake_reference * flakeReference)
+{
+    nix_clear_err(context);
+    try {
+        eval_state->state.resetFileCache();
+        auto lockedFlake = nix::make_ref<nix::flake::LockedFlake>(nix::flake::lockFlake(
+            *flakeSettings->settings, eval_state->state, *flakeReference->flakeRef, *flags->lockFlags));
+        return new nix_locked_flake{lockedFlake};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_locked_flake_free(nix_locked_flake * lockedFlake)
+{
+    delete lockedFlake;
+}
+
+nix_value * nix_locked_flake_get_output_attrs(
+    nix_c_context * context, nix_flake_settings * settings, EvalState * evalState, nix_locked_flake * lockedFlake)
+{
+    nix_clear_err(context);
+    try {
+        auto v = nix_alloc_value(context, evalState);
+        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
+        return v;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libflake-c/nix_api_flake.h

@@ -9,6 +9,7 @@
  * @brief Main entry for the libflake C bindings
  */
 
+#include "nix_api_fetchers.h"
 #include "nix_api_store.h"
 #include "nix_api_util.h"
 #include "nix_api_expr.h"
@@ -18,8 +19,46 @@ extern "C" {
 #endif
 // cffi start
 
+/**
+ * @brief A settings object for configuring the behavior of the nix-flake-c library.
+ * @see nix_flake_settings_new
+ * @see nix_flake_settings_free
+ */
 typedef struct nix_flake_settings nix_flake_settings;
 
+/**
+ * @brief Context and paramaters for parsing a flake reference
+ * @see nix_flake_reference_parse_flags_free
+ * @see nix_flake_reference_parse_string
+ */
+typedef struct nix_flake_reference_parse_flags nix_flake_reference_parse_flags;
+
+/**
+ * @brief A reference to a flake
+ *
+ * A flake reference specifies how to fetch a flake.
+ *
+ * @see nix_flake_reference_from_string
+ * @see nix_flake_reference_free
+ */
+typedef struct nix_flake_reference nix_flake_reference;
+
+/**
+ * @brief Parameters for locking a flake
+ * @see nix_flake_lock_flags_new
+ * @see nix_flake_lock_flags_free
+ * @see nix_flake_lock
+ */
+typedef struct nix_flake_lock_flags nix_flake_lock_flags;
+
+/**
+ * @brief A flake with a suitable lock (file or otherwise)
+ * @see nix_flake_lock
+ * @see nix_locked_flake_free
+ * @see nix_locked_flake_get_output_attrs
+ */
+typedef struct nix_locked_flake nix_locked_flake;
+
 // Function prototypes
 /**
  * Create a nix_flake_settings initialized with default values.
@@ -38,6 +77,8 @@ void nix_flake_settings_free(nix_flake_settings * settings);
  * @brief Initialize a `nix_flake_settings` to contain `builtins.getFlake` and
  * potentially more.
  *
+ * @warning This does not put the eval state in pure mode!
+ *
  * @param[out] context Optional, stores error information
  * @param[in] settings The settings to use for e.g. `builtins.getFlake`
  * @param[in] builder The builder to modify
@@ -45,6 +86,158 @@ void nix_flake_settings_free(nix_flake_settings * settings);
 nix_err nix_flake_settings_add_to_eval_state_builder(
     nix_c_context * context, nix_flake_settings * settings, nix_eval_state_builder * builder);
 
+/**
+ * @brief A new `nix_flake_reference_parse_flags` with defaults
+ */
+nix_flake_reference_parse_flags *
+nix_flake_reference_parse_flags_new(nix_c_context * context, nix_flake_settings * settings);
+
+/**
+ * @brief Deallocate and release the resources associated with a `nix_flake_reference_parse_flags`.
+ * Does not fail.
+ * @param[in] flags the `nix_flake_reference_parse_flags *` to free
+ */
+void nix_flake_reference_parse_flags_free(nix_flake_reference_parse_flags * flags);
+
+/**
+ * @brief Provide a base directory for parsing relative flake references
+ * @param[out] context Optional, stores error information
+ * @param[in] flags The flags to modify
+ * @param[in] baseDirectory The base directory to add
+ * @param[in] baseDirectoryLen The length of baseDirectory
+ * @return NIX_OK on success, NIX_ERR on failure
+ */
+nix_err nix_flake_reference_parse_flags_set_base_directory(
+    nix_c_context * context,
+    nix_flake_reference_parse_flags * flags,
+    const char * baseDirectory,
+    size_t baseDirectoryLen);
+
+/**
+ * @brief A new `nix_flake_lock_flags` with defaults
+ * @param[in] settings Flake settings that may affect the defaults
+ */
+nix_flake_lock_flags * nix_flake_lock_flags_new(nix_c_context * context, nix_flake_settings * settings);
+
+/**
+ * @brief Deallocate and release the resources associated with a `nix_flake_lock_flags`.
+ * Does not fail.
+ * @param[in] settings the `nix_flake_lock_flags *` to free
+ */
+void nix_flake_lock_flags_free(nix_flake_lock_flags * settings);
+
+/**
+ * @brief Put the lock flags in a mode that checks whether the lock is up to date.
+ * @param[out] context Optional, stores error information
+ * @param[in] flags The flags to modify
+ * @return NIX_OK on success, NIX_ERR on failure
+ *
+ * This causes `nix_flake_lock` to fail if the lock needs to be updated.
+ */
+nix_err nix_flake_lock_flags_set_mode_check(nix_c_context * context, nix_flake_lock_flags * flags);
+
+/**
+ * @brief Put the lock flags in a mode that updates the lock file in memory, if needed.
+ * @param[out] context Optional, stores error information
+ * @param[in] flags The flags to modify
+ * @param[in] update Whether to allow updates
+ *
+ * This will cause `nix_flake_lock` to update the lock file in memory, if needed.
+ */
+nix_err nix_flake_lock_flags_set_mode_virtual(nix_c_context * context, nix_flake_lock_flags * flags);
+
+/**
+ * @brief Put the lock flags in a mode that updates the lock file on disk, if needed.
+ * @param[out] context Optional, stores error information
+ * @param[in] flags The flags to modify
+ * @param[in] update Whether to allow updates
+ *
+ * This will cause `nix_flake_lock` to update the lock file on disk, if needed.
+ */
+nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, nix_flake_lock_flags * flags);
+
+/**
+ * @brief Add input overrides to the lock flags
+ * @param[out] context Optional, stores error information
+ * @param[in] flags The flags to modify
+ * @param[in] inputPath The input path to override
+ * @param[in] flakeRef The flake reference to use as the override
+ *
+ * This switches the `flags` to `nix_flake_lock_flags_set_mode_virtual` if not in mode
+ * `nix_flake_lock_flags_set_mode_check`.
+ */
+nix_err nix_flake_lock_flags_add_input_override(
+    nix_c_context * context, nix_flake_lock_flags * flags, const char * inputPath, nix_flake_reference * flakeRef);
+
+/**
+ * @brief Lock a flake, if not already locked.
+ * @param[out] context Optional, stores error information
+ * @param[in] settings The flake (and fetch) settings to use
+ * @param[in] flags The locking flags to use
+ * @param[in] flake The flake to lock
+ */
+nix_locked_flake * nix_flake_lock(
+    nix_c_context * context,
+    nix_fetchers_settings * fetchSettings,
+    nix_flake_settings * settings,
+    EvalState * eval_state,
+    nix_flake_lock_flags * flags,
+    nix_flake_reference * flake);
+
+/**
+ * @brief Deallocate and release the resources associated with a `nix_locked_flake`.
+ * Does not fail.
+ * @param[in] locked_flake the `nix_locked_flake *` to free
+ */
+void nix_locked_flake_free(nix_locked_flake * locked_flake);
+
+/**
+ * @brief Parse a URL-like string into a `nix_flake_reference`.
+ *
+ * @param[out] context **context** – Optional, stores error information
+ * @param[in] fetchSettings **context** – The fetch settings to use
+ * @param[in] flakeSettings **context** – The flake settings to use
+ * @param[in] parseFlags **context** – Specific context and parameters such as base directory
+ *
+ * @param[in] str **input** – The URI-like string to parse
+ * @param[in] strLen **input** – The length of `str`
+ *
+ * @param[out] flakeReferenceOut **result** – The resulting flake reference
+ * @param[in] fragmentCallback **result** – A callback to call with the fragment part of the URL
+ * @param[in] fragmentCallbackUserData **result** – User data to pass to the fragment callback
+ *
+ * @return NIX_OK on success, NIX_ERR on failure
+ */
+nix_err nix_flake_reference_and_fragment_from_string(
+    nix_c_context * context,
+    nix_fetchers_settings * fetchSettings,
+    nix_flake_settings * flakeSettings,
+    nix_flake_reference_parse_flags * parseFlags,
+    const char * str,
+    size_t strLen,
+    nix_flake_reference ** flakeReferenceOut,
+    nix_get_string_callback fragmentCallback,
+    void * fragmentCallbackUserData);
+
+/**
+ * @brief Deallocate and release the resources associated with a `nix_flake_reference`.
+ *
+ * Does not fail.
+ *
+ * @param[in] store the `nix_flake_reference *` to free
+ */
+void nix_flake_reference_free(nix_flake_reference * store);
+
+/**
+ * @brief Get the output attributes of a flake.
+ * @param[out] context Optional, stores error information
+ * @param[in] settings The settings to use
+ * @param[in] locked_flake the flake to get the output attributes from
+ * @return A new nix_value or NULL on failure. Release the `nix_value` with `nix_value_decref`.
+ */
+nix_value * nix_locked_flake_get_output_attrs(
+    nix_c_context * context, nix_flake_settings * settings, EvalState * evalState, nix_locked_flake * lockedFlake);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

src/libflake-c/nix_api_flake_internal.hh

@@ -1,9 +1,32 @@
 #pragma once
+#include <optional>
 
 #include "nix/util/ref.hh"
+#include "nix/flake/flake.hh"
+#include "nix/flake/flakeref.hh"
 #include "nix/flake/settings.hh"
 
 struct nix_flake_settings
 {
     nix::ref<nix::flake::Settings> settings;
 };
+
+struct nix_flake_reference_parse_flags
+{
+    std::optional<nix::Path> baseDirectory;
+};
+
+struct nix_flake_reference
+{
+    nix::ref<nix::FlakeRef> flakeRef;
+};
+
+struct nix_flake_lock_flags
+{
+    nix::ref<nix::flake::LockFlags> lockFlags;
+};
+
+struct nix_locked_flake
+{
+    nix::ref<nix::flake::LockedFlake> lockedFlake;
+};

src/libflake-c/package.nix

@@ -4,6 +4,7 @@
 
   nix-store-c,
   nix-expr-c,
+  nix-fetchers-c,
   nix-flake,
 
   # Configuration Options
@@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: {
   propagatedBuildInputs = [
     nix-expr-c
     nix-store-c
+    nix-fetchers-c
     nix-flake
   ];
 

src/libflake-tests/nix_api_flake.cc

@@ -1,7 +1,6 @@
+#include "nix/util/file-system.hh"
 #include "nix_api_store.h"
-#include "nix_api_store_internal.h"
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix_api_expr.h"
 #include "nix_api_value.h"
 #include "nix_api_flake.h"
@@ -14,7 +13,7 @@
 
 namespace nixC {
 
-TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists)
+TEST_F(nix_api_store_test, nix_api_init_getFlake_exists)
 {
     nix_libstore_init(ctx);
     assert_ctx_ok();
@@ -43,9 +42,342 @@ TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists)
     ASSERT_NE(nullptr, value);
 
     nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value);
+
+    nix_state_free(state);
+
     assert_ctx_ok();
     ASSERT_EQ(NIX_OK, err);
     ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value));
 }
 
+TEST_F(nix_api_store_test, nix_api_flake_reference_not_absolute_no_basedir_fail)
+{
+    nix_libstore_init(ctx);
+    assert_ctx_ok();
+    nix_libexpr_init(ctx);
+    assert_ctx_ok();
+
+    auto settings = nix_flake_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, settings);
+
+    auto fetchSettings = nix_fetchers_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, fetchSettings);
+
+    auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings);
+
+    std::string str(".#legacyPackages.aarch127-unknown...orion");
+    std::string fragment;
+    nix_flake_reference * flakeReference = nullptr;
+    auto r = nix_flake_reference_and_fragment_from_string(
+        ctx, fetchSettings, settings, parseFlags, str.data(), str.size(), &flakeReference, OBSERVE_STRING(fragment));
+
+    ASSERT_NE(NIX_OK, r);
+    ASSERT_EQ(nullptr, flakeReference);
+
+    nix_flake_reference_parse_flags_free(parseFlags);
+}
+
+TEST_F(nix_api_store_test, nix_api_load_flake)
+{
+    auto tmpDir = nix::createTempDir();
+    nix::AutoDelete delTmpDir(tmpDir, true);
+
+    nix::writeFile(tmpDir + "/flake.nix", R"(
+        {
+            outputs = { ... }: {
+                hello = "potato";
+            };
+        }
+    )");
+
+    nix_libstore_init(ctx);
+    assert_ctx_ok();
+    nix_libexpr_init(ctx);
+    assert_ctx_ok();
+
+    auto fetchSettings = nix_fetchers_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, fetchSettings);
+
+    auto settings = nix_flake_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, settings);
+
+    nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store);
+    ASSERT_NE(nullptr, builder);
+    assert_ctx_ok();
+
+    auto state = nix_eval_state_build(ctx, builder);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, state);
+
+    nix_eval_state_builder_free(builder);
+
+    auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, parseFlags);
+
+    auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size());
+    assert_ctx_ok();
+    ASSERT_EQ(NIX_OK, r0);
+
+    std::string fragment;
+    const std::string ref = ".#legacyPackages.aarch127-unknown...orion";
+    nix_flake_reference * flakeReference = nullptr;
+    auto r = nix_flake_reference_and_fragment_from_string(
+        ctx, fetchSettings, settings, parseFlags, ref.data(), ref.size(), &flakeReference, OBSERVE_STRING(fragment));
+    assert_ctx_ok();
+    ASSERT_EQ(NIX_OK, r);
+    ASSERT_NE(nullptr, flakeReference);
+    ASSERT_EQ(fragment, "legacyPackages.aarch127-unknown...orion");
+
+    nix_flake_reference_parse_flags_free(parseFlags);
+
+    auto lockFlags = nix_flake_lock_flags_new(ctx, settings);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockFlags);
+
+    auto lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockedFlake);
+
+    nix_flake_lock_flags_free(lockFlags);
+
+    auto value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    auto helloAttr = nix_get_attr_byname(ctx, value, state, "hello");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, helloAttr);
+
+    std::string helloStr;
+    nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr));
+    assert_ctx_ok();
+    ASSERT_EQ("potato", helloStr);
+
+    nix_value_decref(ctx, value);
+    nix_locked_flake_free(lockedFlake);
+    nix_flake_reference_free(flakeReference);
+    nix_state_free(state);
+    nix_flake_settings_free(settings);
+}
+
+TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
+{
+    nix_libstore_init(ctx);
+    assert_ctx_ok();
+    nix_libexpr_init(ctx);
+    assert_ctx_ok();
+
+    auto tmpDir = nix::createTempDir();
+    nix::AutoDelete delTmpDir(tmpDir, true);
+
+    nix::createDirs(tmpDir + "/b");
+    nix::writeFile(tmpDir + "/b/flake.nix", R"(
+        {
+            outputs = { ... }: {
+                hello = "BOB";
+            };
+        }
+    )");
+
+    nix::createDirs(tmpDir + "/a");
+    nix::writeFile(tmpDir + "/a/flake.nix", R"(
+        {
+            inputs.b.url = ")" + tmpDir + R"(/b";
+            outputs = { b, ... }: {
+                hello = b.hello;
+            };
+        }
+    )");
+
+    nix::createDirs(tmpDir + "/c");
+    nix::writeFile(tmpDir + "/c/flake.nix", R"(
+        {
+            outputs = { ... }: {
+                hello = "Claire";
+            };
+        }
+    )");
+
+    nix_libstore_init(ctx);
+    assert_ctx_ok();
+
+    auto fetchSettings = nix_fetchers_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, fetchSettings);
+
+    auto settings = nix_flake_settings_new(ctx);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, settings);
+
+    nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store);
+    ASSERT_NE(nullptr, builder);
+    assert_ctx_ok();
+
+    auto state = nix_eval_state_build(ctx, builder);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, state);
+
+    nix_eval_state_builder_free(builder);
+
+    auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, parseFlags);
+
+    auto r0 = nix_flake_reference_parse_flags_set_base_directory(ctx, parseFlags, tmpDir.c_str(), tmpDir.size());
+    assert_ctx_ok();
+    ASSERT_EQ(NIX_OK, r0);
+
+    std::string fragment;
+    const std::string ref = "./a";
+    nix_flake_reference * flakeReference = nullptr;
+    auto r = nix_flake_reference_and_fragment_from_string(
+        ctx, fetchSettings, settings, parseFlags, ref.data(), ref.size(), &flakeReference, OBSERVE_STRING(fragment));
+    assert_ctx_ok();
+    ASSERT_EQ(NIX_OK, r);
+    ASSERT_NE(nullptr, flakeReference);
+    ASSERT_EQ(fragment, "");
+
+    // Step 1: Do not update, fails
+
+    auto lockFlags = nix_flake_lock_flags_new(ctx, settings);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockFlags);
+
+    nix_flake_lock_flags_set_mode_check(ctx, lockFlags);
+    assert_ctx_ok();
+
+    // Step 2: Update but do not write, succeeds
+
+    auto lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_err();
+    ASSERT_EQ(nullptr, lockedFlake);
+
+    nix_flake_lock_flags_set_mode_virtual(ctx, lockFlags);
+    assert_ctx_ok();
+
+    lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockedFlake);
+
+    // Get the output attrs
+    auto value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    auto helloAttr = nix_get_attr_byname(ctx, value, state, "hello");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, helloAttr);
+
+    std::string helloStr;
+    nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr));
+    assert_ctx_ok();
+    ASSERT_EQ("BOB", helloStr);
+
+    nix_value_decref(ctx, value);
+    nix_locked_flake_free(lockedFlake);
+
+    // Step 3: Lock was not written, so Step 1 would fail again
+
+    nix_flake_lock_flags_set_mode_check(ctx, lockFlags);
+
+    lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_err();
+    ASSERT_EQ(nullptr, lockedFlake);
+
+    // Step 4: Update and write, succeeds
+
+    nix_flake_lock_flags_set_mode_write_as_needed(ctx, lockFlags);
+    assert_ctx_ok();
+
+    lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockedFlake);
+
+    // Get the output attrs
+    value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    helloAttr = nix_get_attr_byname(ctx, value, state, "hello");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, helloAttr);
+
+    helloStr.clear();
+    nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr));
+    assert_ctx_ok();
+    ASSERT_EQ("BOB", helloStr);
+
+    nix_value_decref(ctx, value);
+    nix_locked_flake_free(lockedFlake);
+
+    // Step 5: Lock was written, so Step 1 would succeed
+
+    nix_flake_lock_flags_set_mode_check(ctx, lockFlags);
+    assert_ctx_ok();
+
+    lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockedFlake);
+
+    // Get the output attrs
+    value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    helloAttr = nix_get_attr_byname(ctx, value, state, "hello");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, helloAttr);
+
+    helloStr.clear();
+    nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr));
+    assert_ctx_ok();
+    ASSERT_EQ("BOB", helloStr);
+
+    nix_value_decref(ctx, value);
+    nix_locked_flake_free(lockedFlake);
+
+    // Step 6: Lock with override, do not write
+
+    nix_flake_lock_flags_set_mode_write_as_needed(ctx, lockFlags);
+    assert_ctx_ok();
+
+    nix_flake_reference * overrideFlakeReference = nullptr;
+    nix_flake_reference_and_fragment_from_string(
+        ctx, fetchSettings, settings, parseFlags, "./c", 3, &overrideFlakeReference, OBSERVE_STRING(fragment));
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, overrideFlakeReference);
+
+    nix_flake_lock_flags_add_input_override(ctx, lockFlags, "b", overrideFlakeReference);
+    assert_ctx_ok();
+
+    lockedFlake = nix_flake_lock(ctx, fetchSettings, settings, state, lockFlags, flakeReference);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, lockedFlake);
+
+    // Get the output attrs
+    value = nix_locked_flake_get_output_attrs(ctx, settings, state, lockedFlake);
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, value);
+
+    helloAttr = nix_get_attr_byname(ctx, value, state, "hello");
+    assert_ctx_ok();
+    ASSERT_NE(nullptr, helloAttr);
+
+    helloStr.clear();
+    nix_get_string(ctx, helloAttr, OBSERVE_STRING(helloStr));
+    assert_ctx_ok();
+    ASSERT_EQ("Claire", helloStr);
+
+    nix_flake_reference_parse_flags_free(parseFlags);
+    nix_flake_lock_flags_free(lockFlags);
+    nix_flake_reference_free(flakeReference);
+    nix_state_free(state);
+    nix_flake_settings_free(settings);
+}
+
 } // namespace nixC

src/libflake/config.cc

@@ -32,7 +32,7 @@ static void writeTrustedList(const TrustedList & trustedList)
 
 void ConfigFile::apply(const Settings & flakeSettings)
 {
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
+    StringSet whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
 
     for (auto & [name, value] : settings) {
 

src/libflake/flake.cc

@@ -307,7 +307,7 @@ static FlakeRef applySelfAttrs(
 {
     auto newRef(ref);
 
-    std::set<std::string> allowedAttrs{"submodules", "lfs"};
+    StringSet allowedAttrs{"submodules", "lfs"};
 
     for (auto & attr : flake.selfAttrs) {
         if (!allowedAttrs.contains(attr.first))
@@ -821,16 +821,16 @@ LockedFlake lockFlake(
                             auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
                             auto outputLockFilePath = *sourcePath / relPath;
 
-                            bool lockFileExists = fs::symlink_exists(outputLockFilePath);
+                            bool lockFileExists = pathExists(outputLockFilePath);
 
                             auto s = chomp(diff);
                             if (lockFileExists) {
                                 if (s.empty())
-                                    warn("updating lock file '%s'", outputLockFilePath);
+                                    warn("updating lock file %s", outputLockFilePath);
                                 else
-                                    warn("updating lock file '%s':\n%s", outputLockFilePath, s);
+                                    warn("updating lock file %s:\n%s", outputLockFilePath, s);
                             } else
-                                warn("creating lock file '%s': \n%s", outputLockFilePath, s);
+                                warn("creating lock file %s: \n%s", outputLockFilePath, s);
 
                             std::optional<std::string> commitMessage = std::nullopt;
 

src/libflake/include/nix/flake/flake.hh

@@ -63,7 +63,7 @@ struct ConfigFile
 };
 
 /**
- * The contents of a flake.nix file.
+ * A flake in context
  */
 struct Flake
 {

src/libmain/common-args.cc

@@ -1,3 +1,5 @@
+#include <nlohmann/json.hpp>
+
 #include "nix/main/common-args.hh"
 #include "nix/util/args/root.hh"
 #include "nix/util/config-global.hh"
@@ -93,5 +95,18 @@ void MixCommonArgs::initialFlagsProcessed()
     pluginsInited();
 }
 
-
+template <typename T, typename>
+void MixPrintJSON::printJSON(const T /* nlohmann::json */ & json)
+{
+    auto suspension = logger->suspend();
+    if (outputPretty) {
+        logger->writeToStdout(json.dump(2));
+    } else {
+        logger->writeToStdout(json.dump());
+    }
 }
+
+template void MixPrintJSON::printJSON(const nlohmann::json & json);
+
+
+} // namespace nix

src/libmain/include/nix/main/common-args.hh

@@ -29,13 +29,69 @@ struct MixDryRun : virtual Args
         addFlag({
             .longName = "dry-run",
             .description = "Show what this command would do without doing it.",
-            //.category = commonArgsCategory,
             .handler = {&dryRun, true},
         });
     }
 };
 
-struct MixJSON : virtual Args
+/**
+ * Commands that can print JSON according to the
+ * `--pretty`/`--no-pretty` flag.
+ *
+ * This is distinct from MixJSON, because for some commands,
+ * JSON outputs is not optional.
+ */
+struct MixPrintJSON : virtual Args
+{
+    bool outputPretty = isatty(STDOUT_FILENO);
+
+    MixPrintJSON()
+    {
+        addFlag({
+            .longName = "pretty",
+            .description =
+                R"(
+                    Print multi-line, indented JSON output for readability.
+
+                    Default: indent if output is to a terminal.
+
+                    This option is only effective when `--json` is also specified.
+                )",
+            .handler = {&outputPretty, true},
+        });
+        addFlag({
+            .longName = "no-pretty",
+            .description =
+                R"(
+                    Print compact JSON output on a single line, even when the output is a terminal.
+                    Some commands may print multiple JSON objects on separate lines.
+
+                    See `--pretty`.
+                )",
+            .handler = {&outputPretty, false},
+        });
+    };
+
+    /**
+     * Print an `nlohmann::json` to stdout
+     *
+     * - respecting `--pretty` / `--no-pretty`.
+     * - suspending the progress bar
+     *
+     * This is a template to avoid accidental coercions from `string` to `json` in the caller,
+     * to avoid mistakenly passing an already serialized JSON to this function.
+     *
+     * It is not recommended to print a JSON string - see the JSON guidelines
+     * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html -
+     * but you _can_ print a sole JSON string by explicitly coercing it to
+     * `nlohmann::json` first.
+     */
+    template <typename T, typename = std::enable_if_t<std::is_same_v<T, nlohmann::json>>>
+    void printJSON(const T & json);
+};
+
+/** Optional JSON support via `--json` flag */
+struct MixJSON : virtual Args, virtual MixPrintJSON
 {
     bool json = false;
 
@@ -44,7 +100,6 @@ struct MixJSON : virtual Args
         addFlag({
             .longName = "json",
             .description = "Produce output in JSON format, suitable for consumption by another program.",
-            //.category = commonArgsCategory,
             .handler = {&json, true},
         });
     }

src/libmain/plugin.cc

@@ -6,6 +6,7 @@
 
 #include "nix/util/config-global.hh"
 #include "nix/util/signals.hh"
+#include "nix/util/file-system.hh"
 
 namespace nix {
 
@@ -18,7 +19,7 @@ struct PluginFilesSetting : public BaseSetting<Paths>
         const Paths & def,
         const std::string & name,
         const std::string & description,
-        const std::set<std::string> & aliases = {})
+        const StringSet & aliases = {})
         : BaseSetting<Paths>(def, true, name, description, aliases)
     {
         options->addSetting(this);
@@ -77,13 +78,13 @@ void initPlugins()
     for (const auto & pluginFile : pluginSettings.pluginFiles.get()) {
         std::vector<std::filesystem::path> pluginFiles;
         try {
-            auto ents = std::filesystem::directory_iterator{pluginFile};
+            auto ents = DirectoryIterator{pluginFile};
             for (const auto & ent : ents) {
                 checkInterrupt();
                 pluginFiles.emplace_back(ent.path());
             }
-        } catch (std::filesystem::filesystem_error & e) {
-            if (e.code() != std::errc::not_a_directory)
+        } catch (SysError & e) {
+            if (e.errNo != ENOTDIR)
                 throw;
             pluginFiles.emplace_back(pluginFile);
         }

src/libstore-c/nix_api_store.cc

@@ -5,6 +5,7 @@
 
 #include "nix/store/path.hh"
 #include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/build-result.hh"
 
 #include "nix/store/globals.hh"
@@ -42,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
         if (!params)
             return new Store{nix::openStore(uri_str)};
 
-        nix::Store::Params params_map;
+        nix::Store::Config::Params params_map;
         for (size_t i = 0; params[i] != nullptr; i++) {
             params_map[params[i][0]] = params[i][1];
         }

src/libstore-test-support/include/nix/store/tests/libstore.hh

@@ -5,6 +5,7 @@
 #include <gmock/gmock.h>
 
 #include "nix/store/store-api.hh"
+#include "nix/store/store-open.hh"
 
 namespace nix {
 

src/libstore-test-support/include/nix/store/tests/nix_api_store.hh

@@ -11,8 +11,6 @@
 #include <filesystem>
 #include <gtest/gtest.h>
 
-namespace fs { using namespace std::filesystem; }
-
 namespace nixC {
 class nix_api_store_test : public nix_api_util_context
 {
@@ -27,10 +25,10 @@ public:
     {
         nix_store_free(store);
 
-        for (auto & path : fs::recursive_directory_iterator(nixDir)) {
-            fs::permissions(path, fs::perms::owner_all);
+        for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) {
+            std::filesystem::permissions(path, std::filesystem::perms::owner_all);
         }
-        fs::remove_all(nixDir);
+        std::filesystem::remove_all(nixDir);
     }
 
     Store * store;
@@ -45,7 +43,7 @@ protected:
         auto tmpl = nix::defaultTempDir() + "/tests_nix-store.";
         for (size_t i = 0; true; ++i) {
             nixDir = tmpl + std::string { i };
-            if (fs::create_directory(nixDir)) break;
+            if (std::filesystem::create_directory(nixDir)) break;
         }
 #else
         // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp

src/libstore-test-support/outputs-spec.cc

@@ -14,8 +14,9 @@ Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
                 return gen::just((OutputsSpec) OutputsSpec::All{});
             case 1:
                 return gen::map(
-                    gen::nonEmpty(gen::container<StringSet>(
-                        gen::map(gen::arbitrary<StorePathName>(), [](StorePathName n) { return n.name; }))),
+                    gen::nonEmpty(
+                        gen::container<StringSet>(
+                            gen::map(gen::arbitrary<StorePathName>(), [](StorePathName n) { return n.name; }))),
                     [](StringSet names) { return (OutputsSpec) OutputsSpec::Names{names}; });
             default:
                 assert(false);

src/libstore-tests/common-protocol.cc

@@ -154,7 +154,7 @@ CHARACTERIZATION_TEST(
 CHARACTERIZATION_TEST(
     set,
     "set",
-    (std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
+    (std::tuple<StringSet, StringSet, StringSet, std::set<StringSet>> {
         { },
         { "" },
         { "", "foo", "bar" },

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -5,7 +5,7 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
+    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
     "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m",
     "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
@@ -26,7 +26,9 @@
       ]
     }
   },
-  "inputSrcs": [],
+  "inputSrcs": [
+    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+  ],
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
     "bin": {

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -15,6 +15,7 @@
     "builder": "/bin/bash",
     "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
     "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8",
+    "exportReferencesGraph": "refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
     "impureEnvVars": "UNICORN",
     "name": "advanced-attributes",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9",
@@ -40,7 +41,9 @@
       ]
     }
   },
-  "inputSrcs": [],
+  "inputSrcs": [
+    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+  ],
   "name": "advanced-attributes",
   "outputs": {
     "out": {

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json

@@ -5,10 +5,10 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
-    "bin": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin",
-    "dev": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev",
-    "out": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs"
+    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
+    "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin",
+    "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
+    "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
   },
   "inputDrvs": {
     "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
@@ -26,17 +26,19 @@
       ]
     }
   },
-  "inputSrcs": [],
+  "inputSrcs": [
+    "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+  ],
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
     "bin": {
-      "path": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin"
+      "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin"
     },
     "dev": {
-      "path": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev"
+      "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev"
     },
     "out": {
-      "path": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs"
+      "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
     }
   },
   "system": "my-system"

src/libstore-tests/data/derivation/ia/advanced-attributes.json

@@ -15,9 +15,10 @@
     "builder": "/bin/bash",
     "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
     "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
+    "exportReferencesGraph": "refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
     "impureEnvVars": "UNICORN",
     "name": "advanced-attributes",
-    "out": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes",
+    "out": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes",
     "preferLocalBuild": "1",
     "requiredSystemFeatures": "rainbow uid-range",
     "system": "my-system"
@@ -38,11 +39,13 @@
       ]
     }
   },
-  "inputSrcs": [],
+  "inputSrcs": [
+    "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+  ],
   "name": "advanced-attributes",
   "outputs": {
     "out": {
-      "path": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes"
+      "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes"
     }
   },
   "system": "my-system"

src/libstore-tests/derivation-advanced-attrs.cc

@@ -99,6 +99,8 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
 
 #undef TEST_ATERM_JSON
 
+using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
+
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
 {
     this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
@@ -106,16 +108,17 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
-        EXPECT_TRUE(!parsedDrv.hasStructuredAttrs());
+        EXPECT_TRUE(!parsedDrv);
 
         EXPECT_EQ(options.additionalSandboxProfile, "");
         EXPECT_EQ(options.noChroot, false);
         EXPECT_EQ(options.impureHostDeps, StringSet{});
         EXPECT_EQ(options.impureEnvVars, StringSet{});
         EXPECT_EQ(options.allowLocalNetworking, false);
+        EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{});
         {
             auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
             ASSERT_TRUE(checksForAllOutputs_ != nullptr);
@@ -140,8 +143,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
         EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
     });
@@ -154,8 +157,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
         EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
     });
@@ -168,10 +171,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
-        EXPECT_TRUE(!parsedDrv.hasStructuredAttrs());
+        EXPECT_TRUE(!parsedDrv);
 
         EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
         EXPECT_EQ(options.noChroot, true);
@@ -192,8 +195,25 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
+
+        EXPECT_EQ(
+            options.exportReferencesGraph,
+            (ExportReferencesMap{
+                {
+                    "refs1",
+                    {
+                        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
+                    },
+                },
+                {
+                    "refs2",
+                    {
+                        "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
+                    },
+                },
+            }));
 
         {
             auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
@@ -225,8 +245,25 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
+
+        EXPECT_EQ(
+            options.exportReferencesGraph,
+            (ExportReferencesMap{
+                {
+                    "refs1",
+                    {
+                        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
+                    },
+                },
+                {
+                    "refs2",
+                    {
+                        "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+                    },
+                },
+            }));
 
         {
             auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks);
@@ -261,23 +298,24 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
-        EXPECT_TRUE(parsedDrv.hasStructuredAttrs());
+        EXPECT_TRUE(parsedDrv);
 
         EXPECT_EQ(options.additionalSandboxProfile, "");
         EXPECT_EQ(options.noChroot, false);
         EXPECT_EQ(options.impureHostDeps, StringSet{});
         EXPECT_EQ(options.impureEnvVars, StringSet{});
         EXPECT_EQ(options.allowLocalNetworking, false);
+        EXPECT_EQ(options.exportReferencesGraph, ExportReferencesMap{});
 
         {
             auto * checksPerOutput_ = std::get_if<1>(&options.outputChecks);
             ASSERT_TRUE(checksPerOutput_ != nullptr);
             auto & checksPerOutput = *checksPerOutput_;
 
-            EXPECT_EQ(checksPerOutput.size(), 0);
+            EXPECT_EQ(checksPerOutput.size(), 0u);
         }
 
         EXPECT_EQ(options.canBuildLocally(*this->store, got), false);
@@ -294,8 +332,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
         EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
     });
@@ -308,8 +346,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
         EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
     });
@@ -322,10 +360,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
 
-        EXPECT_TRUE(parsedDrv.hasStructuredAttrs());
+        EXPECT_TRUE(parsedDrv);
 
         EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
         EXPECT_EQ(options.noChroot, true);
@@ -356,8 +394,25 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
+
+        EXPECT_EQ(
+            options.exportReferencesGraph,
+            (ExportReferencesMap{
+                {
+                    "refs1",
+                    {
+                        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
+                    },
+                },
+                {
+                    "refs2",
+                    {
+                        "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
+                    },
+                },
+            }));
 
         {
             {
@@ -393,8 +448,25 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
 
         auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
 
-        ParsedDerivation parsedDrv(drvPath, got);
-        DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv);
+        auto parsedDrv = StructuredAttrs::tryParse(got.env);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
+
+        EXPECT_EQ(
+            options.exportReferencesGraph,
+            (ExportReferencesMap{
+                {
+                    "refs1",
+                    {
+                        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
+                    },
+                },
+                {
+                    "refs2",
+                    {
+                        "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+                    },
+                },
+            }));
 
         {
             {

src/libstore-tests/machines.cc

@@ -163,8 +163,8 @@ TEST(machines, getMachinesWithIncorrectFormat) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReference) {
-    auto path = fs::weakly_canonical(getUnitTestData() / "machines/valid");
-    ASSERT_TRUE(fs::exists(path));
+    auto path = std::filesystem::weakly_canonical(getUnitTestData() / "machines/valid");
+    ASSERT_TRUE(std::filesystem::exists(path));
 
     auto actual = Machine::parseConfig({}, "@" + path.string());
     ASSERT_THAT(actual, SizeIs(3));
@@ -174,22 +174,22 @@ TEST(machines, getMachinesWithCorrectFileReference) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {
-    fs::path path = "/dev/null";
-    ASSERT_TRUE(fs::exists(path));
+    std::filesystem::path path = "/dev/null";
+    ASSERT_TRUE(std::filesystem::exists(path));
 
     auto actual = Machine::parseConfig({}, "@" + path.string());
     ASSERT_THAT(actual, SizeIs(0));
 }
 
 TEST(machines, getMachinesWithIncorrectFileReference) {
-    auto path = fs::weakly_canonical("/not/a/file");
-    ASSERT_TRUE(!fs::exists(path));
+    auto path = std::filesystem::weakly_canonical("/not/a/file");
+    ASSERT_TRUE(!std::filesystem::exists(path));
     auto actual = Machine::parseConfig({}, "@" + path.string());
     ASSERT_THAT(actual, SizeIs(0));
 }
 
 TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {
     EXPECT_THROW(
-        Machine::parseConfig({}, "@" + fs::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()),
+        Machine::parseConfig({}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()),
         FormatError);
 }

src/libstore-tests/outputs-spec.cc

@@ -7,7 +7,7 @@
 namespace nix {
 
 TEST(OutputsSpec, no_empty_names) {
-    ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, "");
+    ASSERT_DEATH(OutputsSpec::Names { StringSet { } }, "");
 }
 
 #define TEST_DONT_PARSE(NAME, STR)           \

src/libstore-tests/path-info.cc

@@ -93,7 +93,7 @@ TEST_F(PathInfoTest, PathInfo_full_shortRefs) {
     ValidPathInfo it = makeFullKeyed(*store, true);
     // it.references = unkeyed.references;
     auto refs = it.shortRefs();
-    ASSERT_EQ(refs.size(), 2);
+    ASSERT_EQ(refs.size(), 2u);
     ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar");
     ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo");
 }

src/libstore-tests/serve-protocol.cc

@@ -374,7 +374,7 @@ VERSIONED_CHARACTERIZATION_TEST(
     set,
     "set",
     defaultVersion,
-    (std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
+    (std::tuple<StringSet, StringSet, StringSet, std::set<StringSet>> {
         { },
         { "" },
         { "", "foo", "bar" },

src/libstore-tests/worker-protocol.cc

@@ -574,7 +574,7 @@ VERSIONED_CHARACTERIZATION_TEST(
     set,
     "set",
     defaultVersion,
-    (std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
+    (std::tuple<StringSet, StringSet, StringSet, std::set<StringSet>> {
         { },
         { "" },
         { "", "foo", "bar" },
@@ -685,7 +685,7 @@ TEST_F(WorkerProtoTest, handshake_features)
     toClient.create();
     toServer.create();
 
-    std::tuple<WorkerProto::Version, std::set<WorkerProto::Feature>> clientResult;
+    std::tuple<WorkerProto::Version, WorkerProto::FeatureSet> clientResult;
 
     auto clientThread = std::thread([&]() {
         FdSink out { toServer.writeSide.get() };
@@ -702,8 +702,8 @@ TEST_F(WorkerProtoTest, handshake_features)
     clientThread.join();
 
     EXPECT_EQ(clientResult, daemonResult);
-    EXPECT_EQ(std::get<0>(clientResult), 123);
-    EXPECT_EQ(std::get<1>(clientResult), std::set<WorkerProto::Feature>({"bar", "xyzzy"}));
+    EXPECT_EQ(std::get<0>(clientResult), 123u);
+    EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"}));
 }
 
 /// Has to be a `BufferedSink` for handshake.

src/libstore/binary-cache-store.cc

@@ -24,13 +24,21 @@
 
 namespace nix {
 
-BinaryCacheStore::BinaryCacheStore(const Params & params)
-    : BinaryCacheStoreConfig(params)
-    , Store(params)
+BinaryCacheStore::BinaryCacheStore(Config & config)
+    : config{config}
 {
-    if (secretKeyFile != "")
-        signer = std::make_unique<LocalSigner>(
-            SecretKey { readFile(secretKeyFile) });
+    if (config.secretKeyFile != "")
+        signers.push_back(std::make_unique<LocalSigner>(
+            SecretKey { readFile(config.secretKeyFile) }));
+
+    if (config.secretKeyFiles != "") {
+        std::stringstream ss(config.secretKeyFiles);
+        Path keyPath;
+        while (std::getline(ss, keyPath, ',')) {
+            signers.push_back(std::make_unique<LocalSigner>(
+                SecretKey { readFile(keyPath) }));
+        }
+    }
 
     StringSink sink;
     sink << narVersionMagic1;
@@ -53,9 +61,9 @@ void BinaryCacheStore::init()
                     throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
                         getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery.setDefault(value == "1");
+                config.wantMassQuery.setDefault(value == "1");
             } else if (name == "Priority") {
-                priority.setDefault(std::stoi(value));
+                config.priority.setDefault(std::stoi(value));
             }
         }
     }
@@ -147,7 +155,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
-    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
+    auto compressionSink = makeCompressionSink(
+        config.compression,
+        teeSinkCompressed,
+        config.parallelCompression,
+        config.compressionLevel);
     TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
     TeeSource teeSource { narSource, teeSinkUncompressed };
     narAccessor = makeNarAccessor(teeSource);
@@ -159,17 +171,17 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
     auto info = mkInfo(narHashSink.finish());
     auto narInfo = make_ref<NarInfo>(info);
-    narInfo->compression = compression;
+    narInfo->compression = config.compression;
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
     narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
-                   + (compression == "xz" ? ".xz" :
-           compression == "bzip2" ? ".bz2" :
-           compression == "zstd" ? ".zst" :
-           compression == "lzip" ? ".lzip" :
-           compression == "lz4" ? ".lz4" :
-           compression == "br" ? ".br" :
+                   + (config.compression == "xz" ? ".xz" :
+           config.compression == "bzip2" ? ".bz2" :
+           config.compression == "zstd" ? ".zst" :
+           config.compression == "lzip" ? ".lzip" :
+           config.compression == "lz4" ? ".lz4" :
+           config.compression == "br" ? ".br" :
            "");
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
@@ -191,7 +203,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
 
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
-    if (writeNARListing) {
+    if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
             {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
@@ -203,7 +215,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     /* Optionally maintain an index of DWARF debug info files
        consisting of JSON files named 'debuginfo/<build-id>' that
        specify the NAR file and member containing the debug info. */
-    if (writeDebugInfo) {
+    if (config.writeDebugInfo) {
 
         CanonPath buildIdDir("lib/debug/.build-id");
 
@@ -270,9 +282,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     stats.narWriteCompressedBytes += fileSize;
     stats.narWriteCompressionTimeMs += duration;
 
-    /* Atomically write the NAR info file.*/
-    if (signer) narInfo->sign(*this, *signer);
+    narInfo->sign(*this, signers);
 
+    /* Atomically write the NAR info file.*/
     writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
@@ -515,7 +527,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
 
 ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
 }
 
 void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)

src/libstore/build/drv-output-substitution-goal.cc

@@ -3,6 +3,7 @@
 #include "nix/store/build/worker.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/util/callback.hh"
+#include "nix/store/store-open.hh"
 
 namespace nix {
 
@@ -11,7 +12,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
     Worker & worker,
     RepairFlag repair,
     std::optional<ContentAddress> ca)
-    : Goal(worker, DerivedPath::Opaque { StorePath::dummy })
+    : Goal(worker)
     , id(id)
 {
     name = fmt("substitution of '%s'", id.to_string());
@@ -87,6 +88,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
 
         bool failed = false;
 
+        Goals waitees;
+
         for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
             if (depId != id) {
                 if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -103,13 +106,13 @@ Goal::Co DrvOutputSubstitutionGoal::init()
                     failed = true;
                     break;
                 }
-                addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+                waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
             }
         }
 
         if (failed) continue;
 
-        co_return realisationFetched(outputInfo, sub);
+        co_return realisationFetched(std::move(waitees), outputInfo, sub);
     }
 
     /* None left.  Terminate this goal and let someone else deal
@@ -127,10 +130,10 @@ Goal::Co DrvOutputSubstitutionGoal::init()
     co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
 }
 
-Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub) {
-    addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+Goal::Co DrvOutputSubstitutionGoal::realisationFetched(Goals waitees, std::shared_ptr<const Realisation> outputInfo, nix::ref<nix::Store> sub) {
+    waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
-    if (!waitees.empty()) co_await Suspend{};
+    co_await await(std::move(waitees));
 
     trace("output path substituted");
 

src/libstore/build/goal.cc

@@ -132,38 +132,18 @@ void addToWeakGoals(WeakGoals & goals, GoalPtr p)
     goals.insert(p);
 }
 
-
-void Goal::addWaitee(GoalPtr waitee)
+Co Goal::await(Goals new_waitees)
 {
-    waitees.insert(waitee);
-    addToWeakGoals(waitee->waiters, shared_from_this());
-}
-
-
-void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
-{
-    assert(waitees.count(waitee));
-    waitees.erase(waitee);
-
-    trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
-
-    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
-
-    if (result == ecNoSubstituters) ++nrNoSubstituters;
-
-    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
-
-    if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
-
-        /* If we failed and keepGoing is not set, we remove all
-           remaining waitees. */
-        for (auto & goal : waitees) {
-            goal->waiters.extract(shared_from_this());
+    assert(waitees.empty());
+    if (!new_waitees.empty()) {
+        waitees = std::move(new_waitees);
+        for (auto waitee : waitees) {
+            addToWeakGoals(waitee->waiters, shared_from_this());
         }
-        waitees.clear();
-
-        worker.wakeUp(shared_from_this());
+        co_await Suspend{};
+        assert(waitees.empty());
     }
+    co_return Return{};
 }
 
 Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
@@ -183,7 +163,32 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
 
     for (auto & i : waiters) {
         GoalPtr goal = i.lock();
-        if (goal) goal->waiteeDone(shared_from_this(), result);
+        if (goal) {
+            auto me = shared_from_this();
+            assert(goal->waitees.count(me));
+            goal->waitees.erase(me);
+
+            goal->trace(fmt("waitee '%s' done; %d left", name, goal->waitees.size()));
+
+            if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++goal->nrFailed;
+
+            if (result == ecNoSubstituters) ++goal->nrNoSubstituters;
+
+            if (result == ecIncompleteClosure) ++goal->nrIncompleteClosure;
+
+            if (goal->waitees.empty()) {
+                worker.wakeUp(goal);
+            } else if (result == ecFailed && !settings.keepGoing) {
+                /* If we failed and keepGoing is not set, we remove all
+                   remaining waitees. */
+                for (auto & g : goal->waitees) {
+                    g->waiters.extract(goal);
+                }
+                goal->waitees.clear();
+
+                worker.wakeUp(goal);
+            }
+        }
     }
     waiters.clear();
     worker.removeGoal(shared_from_this());
@@ -215,5 +220,22 @@ void Goal::work()
     assert(top_co || exitCode != ecBusy);
 }
 
+Goal::Co Goal::yield() {
+    worker.wakeUp(shared_from_this());
+    co_await Suspend{};
+    co_return Return{};
+}
+
+Goal::Co Goal::waitForAWhile() {
+    worker.waitForAWhile(shared_from_this());
+    co_await Suspend{};
+    co_return Return{};
+}
+
+Goal::Co Goal::waitForBuildSlot() {
+    worker.waitForBuildSlot(shared_from_this());
+    co_await Suspend{};
+    co_return Return{};
+}
 
 }

src/libstore/build/substitution-goal.cc

@@ -1,4 +1,5 @@
 #include "nix/store/build/worker.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/store/nar-info.hh"
 #include "nix/util/finally.hh"
@@ -11,7 +12,7 @@
 namespace nix {
 
 PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
-    : Goal(worker, DerivedPath::Opaque { storePath })
+    : Goal(worker)
     , storePath(storePath)
     , repair(repair)
     , ca(ca)
@@ -133,20 +134,22 @@ Goal::Co PathSubstitutionGoal::init()
         /* Bail out early if this substituter lacks a valid
            signature. LocalStore::addToStore() also checks for this, but
            only after we've downloaded the path. */
-        if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
+        if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info))
         {
             warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
                 worker.store.printStorePath(storePath), sub->getUri());
             continue;
         }
 
+        Goals waitees;
+
         /* To maintain the closure invariant, we first have to realise the
            paths referenced by this one. */
         for (auto & i : info->references)
             if (i != storePath) /* ignore self-references */
-                addWaitee(worker.makePathSubstitutionGoal(i));
+                waitees.insert(worker.makePathSubstitutionGoal(i));
 
-        if (!waitees.empty()) co_await Suspend{};
+        co_await await(std::move(waitees));
 
         // FIXME: consider returning boolean instead of passing in reference
         bool out = false; // is mutated by tryToRun
@@ -184,11 +187,15 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
     }
 
     for (auto & i : info->references)
-        if (i != storePath) /* ignore self-references */
-            assert(worker.store.isValidPath(i));
+         /* ignore self-references */
+        if (i != storePath) {
+            if (!worker.store.isValidPath(i)) {
+                throw Error("reference '%s' of path '%s' is not a valid path",
+                            worker.store.printStorePath(i), worker.store.printStorePath(storePath));
+            }
+        }
 
-    worker.wakeUp(shared_from_this());
-    co_await Suspend{};
+    co_await yield();
 
     trace("trying to run");
 
@@ -196,8 +203,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
        if maxSubstitutionJobs == 0, we still allow a substituter to run. This
        prevents infinite waiting. */
     while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
-        worker.waitForBuildSlot(shared_from_this());
-        co_await Suspend{};
+        co_await waitForBuildSlot();
     }
 
     auto maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
@@ -222,7 +228,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
             PushActivity pact(act.id);
 
             copyStorePath(*sub, worker.store,
-                subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+                subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {

src/libstore/build/worker.cc

@@ -5,7 +5,6 @@
 #include "nix/store/build/drv-output-substitution-goal.hh"
 #include "nix/store/build/derivation-goal.hh"
 #ifndef _WIN32 // TODO Enable building on Windows
-#  include "nix/store/build/local-derivation-goal.hh"
 #  include "nix/store/build/hook-instance.hh"
 #endif
 #include "nix/util/signals.hh"
@@ -65,13 +64,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
     const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return
-#ifndef _WIN32 // TODO Enable building on Windows
-            dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
-            :
-#endif
-            std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+        return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -79,13 +72,7 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
     const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
 {
     return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
-        return
-#ifndef _WIN32 // TODO Enable building on Windows
-            dynamic_cast<LocalStore *>(&store)
-            ? std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
-            :
-#endif
-            std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+        return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
     });
 }
 
@@ -524,7 +511,7 @@ bool Worker::pathContentsGood(const StorePath & path)
         res = false;
     else {
         auto current = hashPath(
-            {store.getFSAccessor(), CanonPath(store.printStorePath(path))},
+            {store.getFSAccessor(), CanonPath(path.to_string())},
             FileIngestionMethod::NixArchive, info->narHash.algo).first;
         Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current;
@@ -552,4 +539,9 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
     return subGoal;
 }
 
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
+{
+    return subGoal;
+}
+
 }

src/libstore/builtins/buildenv.cc

@@ -1,4 +1,5 @@
 #include "nix/store/builtins/buildenv.hh"
+#include "nix/store/builtins.hh"
 #include "nix/store/derivations.hh"
 #include "nix/util/signals.hh"
 
@@ -18,12 +19,12 @@ struct State
 /* For each activated package, create symlinks */
 static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
-    std::filesystem::directory_iterator srcFiles;
+    DirectoryIterator srcFiles;
 
     try {
-        srcFiles = std::filesystem::directory_iterator{srcDir};
-    } catch (std::filesystem::filesystem_error & e) {
-        if (e.code() == std::errc::not_a_directory) {
+        srcFiles = DirectoryIterator{srcDir};
+    } catch (SysError & e) {
+        if (e.errNo == ENOTDIR) {
             warn("not including '%s' in the user environment because it's not a directory", srcDir);
             return;
         }
@@ -123,7 +124,7 @@ void buildProfile(const Path & out, Packages && pkgs)
 {
     State state;
 
-    std::set<Path> done, postponed;
+    PathSet done, postponed;
 
     auto addPkg = [&](const Path & pkgDir, int priority) {
         if (!done.insert(pkgDir).second) return;
@@ -157,7 +158,7 @@ void buildProfile(const Path & out, Packages && pkgs)
      */
     auto priorityCounter = 1000;
     while (!postponed.empty()) {
-        std::set<Path> pkgDirs;
+        PathSet pkgDirs;
         postponed.swap(pkgDirs);
         for (const auto & pkgDir : pkgDirs)
             addPkg(pkgDir, priorityCounter++);
@@ -166,17 +167,15 @@ void buildProfile(const Path & out, Packages && pkgs)
     debug("created %d symlinks in user environment", state.symlinks);
 }
 
-void builtinBuildenv(
-    const BasicDerivation & drv,
-    const std::map<std::string, Path> & outputs)
+static void builtinBuildenv(const BuiltinBuilderContext & ctx)
 {
     auto getAttr = [&](const std::string & name) {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        auto i = ctx.drv.env.find(name);
+        if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
-    auto out = outputs.at("out");
+    auto out = ctx.outputs.at("out");
     createDirs(out);
 
     /* Convert the stuff we get from the environment back into a
@@ -203,4 +202,6 @@ void builtinBuildenv(
     createSymlink(getAttr("manifest"), out + "/manifest.nix");
 }
 
+static RegisterBuiltinBuilder registerBuildenv("buildenv", builtinBuildenv);
+
 }

src/libstore/builtins/fetchurl.cc

@@ -6,33 +6,29 @@
 
 namespace nix {
 
-void builtinFetchurl(
-    const BasicDerivation & drv,
-    const std::map<std::string, Path> & outputs,
-    const std::string & netrcData,
-    const std::string & caFileData)
+static void builtinFetchurl(const BuiltinBuilderContext & ctx)
 {
     /* Make the host's netrc data available. Too bad curl requires
        this to be stored in a file. It would be nice if we could just
        pass a pointer to the data. */
-    if (netrcData != "") {
+    if (ctx.netrcData != "") {
         settings.netrcFile = "netrc";
-        writeFile(settings.netrcFile, netrcData, 0600);
+        writeFile(settings.netrcFile, ctx.netrcData, 0600);
     }
 
     settings.caFile = "ca-certificates.crt";
-    writeFile(settings.caFile, caFileData, 0600);
+    writeFile(settings.caFile, ctx.caFileData, 0600);
 
-    auto out = get(drv.outputs, "out");
+    auto out = get(ctx.drv.outputs, "out");
     if (!out)
         throw Error("'builtin:fetchurl' requires an 'out' output");
 
-    if (!(drv.type().isFixed() || drv.type().isImpure()))
+    if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure()))
         throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation");
 
-    auto storePath = outputs.at("out");
-    auto mainUrl = drv.env.at("url");
-    bool unpack = getOr(drv.env, "unpack", "") == "1";
+    auto storePath = ctx.outputs.at("out");
+    auto mainUrl = ctx.drv.env.at("url");
+    bool unpack = getOr(ctx.drv.env, "unpack", "") == "1";
 
     /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */
@@ -56,8 +52,8 @@ void builtinFetchurl(
         else
             writeFile(storePath, *source);
 
-        auto executable = drv.env.find("executable");
-        if (executable != drv.env.end() && executable->second == "1") {
+        auto executable = ctx.drv.env.find("executable");
+        if (executable != ctx.drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
                 throw SysError("making '%1%' executable", storePath);
         }
@@ -79,4 +75,6 @@ void builtinFetchurl(
     fetch(mainUrl);
 }
 
+static RegisterBuiltinBuilder registerFetchurl("fetchurl", builtinFetchurl);
+
 }

src/libstore/builtins/unpack-channel.cc

@@ -3,23 +3,19 @@
 
 namespace nix {
 
-namespace fs { using namespace std::filesystem; }
-
-void builtinUnpackChannel(
-    const BasicDerivation & drv,
-    const std::map<std::string, Path> & outputs)
+static void builtinUnpackChannel(const BuiltinBuilderContext & ctx)
 {
     auto getAttr = [&](const std::string & name) -> const std::string & {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        auto i = ctx.drv.env.find(name);
+        if (i == ctx.drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
-    fs::path out{outputs.at("out")};
+    std::filesystem::path out{ctx.outputs.at("out")};
     auto & channelName = getAttr("channelName");
     auto & src = getAttr("src");
 
-    if (fs::path{channelName}.filename().string() != channelName) {
+    if (std::filesystem::path{channelName}.filename().string() != channelName) {
         throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName);
     }
 
@@ -29,23 +25,21 @@ void builtinUnpackChannel(
 
     size_t fileCount;
     std::string fileName;
-    try {
-        auto entries = fs::directory_iterator{out};
-        fileName = entries->path().string();
-        fileCount = std::distance(fs::begin(entries), fs::end(entries));
-    } catch (fs::filesystem_error &) {
-        throw SysError("failed to read directory %1%", out.string());
-    }
+    auto entries = DirectoryIterator{out};
+    fileName = entries->path().string();
+    fileCount = std::distance(entries.begin(), entries.end());
 
     if (fileCount != 1)
         throw Error("channel tarball '%s' contains more than one file", src);
 
     auto target = out / channelName;
     try {
-        fs::rename(fileName, target);
-    } catch (fs::filesystem_error &) {
+        std::filesystem::rename(fileName, target);
+    } catch (std::filesystem::filesystem_error &) {
         throw SysError("failed to rename %1% to %2%", fileName, target.string());
     }
 }
 
+static RegisterBuiltinBuilder registerUnpackChannel("unpack-channel", builtinUnpackChannel);
+
 }

src/libstore/common-ssh-store-config.cc

@@ -28,7 +28,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_
 {
 }
 
-SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD)
+SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const
 {
     return {
         host,

src/libstore/derivation-options.cc

@@ -1,46 +1,124 @@
 #include "nix/store/derivation-options.hh"
 #include "nix/util/json-utils.hh"
 #include "nix/store/parsed-derivations.hh"
+#include "nix/store/derivations.hh"
+#include "nix/store/store-api.hh"
 #include "nix/util/types.hh"
 #include "nix/util/util.hh"
+
 #include <optional>
 #include <string>
 #include <variant>
+#include <regex>
 
 namespace nix {
 
+static std::optional<std::string>
+getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
+{
+    if (parsed) {
+        auto i = parsed->structuredAttrs.find(name);
+        if (i == parsed->structuredAttrs.end())
+            return {};
+        else {
+            if (!i->is_string())
+                throw Error("attribute '%s' of must be a string", name);
+            return i->get<std::string>();
+        }
+    } else {
+        auto i = env.find(name);
+        if (i == env.end())
+            return {};
+        else
+            return i->second;
+    }
+}
+
+static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def)
+{
+    if (parsed) {
+        auto i = parsed->structuredAttrs.find(name);
+        if (i == parsed->structuredAttrs.end())
+            return def;
+        else {
+            if (!i->is_boolean())
+                throw Error("attribute '%s' must be a Boolean", name);
+            return i->get<bool>();
+        }
+    } else {
+        auto i = env.find(name);
+        if (i == env.end())
+            return def;
+        else
+            return i->second == "1";
+    }
+}
+
+static std::optional<Strings>
+getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
+{
+    if (parsed) {
+        auto i = parsed->structuredAttrs.find(name);
+        if (i == parsed->structuredAttrs.end())
+            return {};
+        else {
+            if (!i->is_array())
+                throw Error("attribute '%s' must be a list of strings", name);
+            Strings res;
+            for (auto j = i->begin(); j != i->end(); ++j) {
+                if (!j->is_string())
+                    throw Error("attribute '%s' must be a list of strings", name);
+                res.push_back(j->get<std::string>());
+            }
+            return res;
+        }
+    } else {
+        auto i = env.find(name);
+        if (i == env.end())
+            return {};
+        else
+            return tokenizeString<Strings>(i->second);
+    }
+}
+
+static std::optional<StringSet>
+getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
+{
+    auto ss = getStringsAttr(env, parsed, name);
+    return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
+}
+
 using OutputChecks = DerivationOptions::OutputChecks;
 
 using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
 
-DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn)
+DerivationOptions
+DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
 {
     DerivationOptions defaults = {};
 
-    auto structuredAttrs = parsed.structuredAttrs.get();
-
-    if (shouldWarn && structuredAttrs) {
-        if (get(*structuredAttrs, "allowedReferences")) {
+    if (shouldWarn && parsed) {
+        if (get(parsed->structuredAttrs, "allowedReferences")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
         }
-        if (get(*structuredAttrs, "allowedRequisites")) {
+        if (get(parsed->structuredAttrs, "allowedRequisites")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
         }
-        if (get(*structuredAttrs, "disallowedRequisites")) {
+        if (get(parsed->structuredAttrs, "disallowedRequisites")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
         }
-        if (get(*structuredAttrs, "disallowedReferences")) {
+        if (get(parsed->structuredAttrs, "disallowedReferences")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
         }
-        if (get(*structuredAttrs, "maxSize")) {
+        if (get(parsed->structuredAttrs, "maxSize")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
         }
-        if (get(*structuredAttrs, "maxClosureSize")) {
+        if (get(parsed->structuredAttrs, "maxClosureSize")) {
             warn(
                 "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
         }
@@ -48,9 +126,9 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
 
     return {
         .outputChecks = [&]() -> OutputChecksVariant {
-            if (auto structuredAttrs = parsed.structuredAttrs.get()) {
+            if (parsed) {
                 std::map<std::string, OutputChecks> res;
-                if (auto outputChecks = get(*structuredAttrs, "outputChecks")) {
+                if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) {
                     for (auto & [outputName, output] : getObject(*outputChecks)) {
                         OutputChecks checks;
 
@@ -87,10 +165,10 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
                 return OutputChecks{
                     // legacy non-structured-attributes case
                     .ignoreSelfRefs = true,
-                    .allowedReferences = parsed.getStringSetAttr("allowedReferences"),
-                    .disallowedReferences = parsed.getStringSetAttr("disallowedReferences").value_or(StringSet{}),
-                    .allowedRequisites = parsed.getStringSetAttr("allowedRequisites"),
-                    .disallowedRequisites = parsed.getStringSetAttr("disallowedRequisites").value_or(StringSet{}),
+                    .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
+                    .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
+                    .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
+                    .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
                 };
             }
         }(),
@@ -98,8 +176,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
             [&] {
                 std::map<std::string, bool> res;
 
-                if (auto structuredAttrs = parsed.structuredAttrs.get()) {
-                    if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
+                if (parsed) {
+                    if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) {
                         for (auto & [outputName, output] : getObject(*udr)) {
                             if (!output.is_boolean())
                                 throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
@@ -113,8 +191,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
         .passAsFile =
             [&] {
                 StringSet res;
-                if (auto * passAsFileString = get(parsed.drv.env, "passAsFile")) {
-                    if (parsed.hasStructuredAttrs()) {
+                if (auto * passAsFileString = get(env, "passAsFile")) {
+                    if (parsed) {
                         if (shouldWarn) {
                             warn(
                                 "'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file");
@@ -125,16 +203,44 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
                 }
                 return res;
             }(),
+        .exportReferencesGraph =
+            [&] {
+                std::map<std::string, StringSet> ret;
+
+                if (parsed) {
+                    auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
+                    if (!e || !e->is_object())
+                        return ret;
+                    for (auto & [key, storePathsJson] : getObject(*e)) {
+                        ret.insert_or_assign(key, storePathsJson);
+                    }
+                } else {
+                    auto s = getOr(env, "exportReferencesGraph", "");
+                    Strings ss = tokenizeString<Strings>(s);
+                    if (ss.size() % 2 != 0)
+                        throw Error("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
+                    for (Strings::iterator i = ss.begin(); i != ss.end();) {
+                        auto fileName = std::move(*i++);
+                        static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
+                        if (!std::regex_match(fileName, regex))
+                            throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
+
+                        auto & storePathS = *i++;
+                        ret.insert_or_assign(std::move(fileName), StringSet{storePathS});
+                    }
+                }
+                return ret;
+            }(),
         .additionalSandboxProfile =
-            parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile),
-        .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot),
-        .impureHostDeps = parsed.getStringSetAttr("__impureHostDeps").value_or(defaults.impureHostDeps),
-        .impureEnvVars = parsed.getStringSetAttr("impureEnvVars").value_or(defaults.impureEnvVars),
-        .allowLocalNetworking = parsed.getBoolAttr("__darwinAllowLocalNetworking", defaults.allowLocalNetworking),
+            getStringAttr(env, parsed, "__sandboxProfile").value_or(defaults.additionalSandboxProfile),
+        .noChroot = getBoolAttr(env, parsed, "__noChroot", defaults.noChroot),
+        .impureHostDeps = getStringSetAttr(env, parsed, "__impureHostDeps").value_or(defaults.impureHostDeps),
+        .impureEnvVars = getStringSetAttr(env, parsed, "impureEnvVars").value_or(defaults.impureEnvVars),
+        .allowLocalNetworking = getBoolAttr(env, parsed, "__darwinAllowLocalNetworking", defaults.allowLocalNetworking),
         .requiredSystemFeatures =
-            parsed.getStringSetAttr("requiredSystemFeatures").value_or(defaults.requiredSystemFeatures),
-        .preferLocalBuild = parsed.getBoolAttr("preferLocalBuild", defaults.preferLocalBuild),
-        .allowSubstitutes = parsed.getBoolAttr("allowSubstitutes", defaults.allowSubstitutes),
+            getStringSetAttr(env, parsed, "requiredSystemFeatures").value_or(defaults.requiredSystemFeatures),
+        .preferLocalBuild = getBoolAttr(env, parsed, "preferLocalBuild", defaults.preferLocalBuild),
+        .allowSubstitutes = getBoolAttr(env, parsed, "allowSubstitutes", defaults.allowSubstitutes),
     };
 }
 
@@ -159,7 +265,7 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivatio
         return false;
 
     for (auto & feature : getRequiredSystemFeatures(drv))
-        if (!localStore.systemFeatures.get().count(feature))
+        if (!localStore.config.systemFeatures.get().count(feature))
             return false;
 
     return true;

src/libstore/derivations.cc

@@ -1052,49 +1052,36 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
 {
-    std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
-
-    std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
-    accum = [&](auto & inputDrv, auto & node) {
-        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
-            if (outputPath) {
-                inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
-                if (auto p = get(node.childMap, outputName))
-                    accum(*outputPath, *p);
+    return tryResolve(
+        store,
+        [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
+            try {
+                return resolveDerivedPath(store, SingleDerivedPath::Built{drvPath, outputName}, evalStore);
+            } catch (Error &) {
+                return std::nullopt;
             }
-        }
-    };
-
-    for (auto & [inputDrv, node] : inputDrvs.map)
-        accum(inputDrv, node);
-
-    return tryResolve(store, inputDrvOutputs);
+        });
 }
 
 static bool tryResolveInput(
     Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
     const DownstreamPlaceholder * placeholderOpt,
-    const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode,
-    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs)
+    ref<const SingleDerivedPath> drvPath, const DerivedPathMap<StringSet>::ChildNode & inputNode,
+    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)> queryResolutionChain)
 {
-    auto getOutput = [&](const std::string & outputName) {
-        auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName });
-        if (!actualPathOpt)
-            warn("output %s of input %s missing, aborting the resolving",
-                outputName,
-                store.printStorePath(inputDrv)
-            );
-        return actualPathOpt;
-    };
-
     auto getPlaceholder = [&](const std::string & outputName) {
         return placeholderOpt
             ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName)
-            : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName);
+            : [&]{
+                auto * p = std::get_if<SingleDerivedPath::Opaque>(&drvPath->raw());
+                // otherwise we should have had a placeholder to build-upon already
+                assert(p);
+                return DownstreamPlaceholder::unknownCaOutput(p->path, outputName);
+            }();
     };
 
     for (auto & outputName : inputNode.value) {
-        auto actualPathOpt = getOutput(outputName);
+        auto actualPathOpt = queryResolutionChain(drvPath, outputName);
         if (!actualPathOpt) return false;
         auto actualPath = *actualPathOpt;
         if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -1106,13 +1093,12 @@ static bool tryResolveInput(
     }
 
     for (auto & [outputName, childNode] : inputNode.childMap) {
-        auto actualPathOpt = getOutput(outputName);
-        if (!actualPathOpt) return false;
-        auto actualPath = *actualPathOpt;
         auto nextPlaceholder = getPlaceholder(outputName);
         if (!tryResolveInput(store, inputSrcs, inputRewrites,
-            &nextPlaceholder, actualPath, childNode,
-            inputDrvOutputs))
+            &nextPlaceholder,
+            make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{drvPath, outputName}),
+            childNode,
+            queryResolutionChain))
             return false;
     }
     return true;
@@ -1120,7 +1106,7 @@ static bool tryResolveInput(
 
 std::optional<BasicDerivation> Derivation::tryResolve(
     Store & store,
-    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
+    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)> queryResolutionChain) const
 {
     BasicDerivation resolved { *this };
 
@@ -1129,7 +1115,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
 
     for (auto & [inputDrv, inputNode] : inputDrvs.map)
         if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites,
-            nullptr, inputDrv, inputNode, inputDrvOutputs))
+            nullptr, make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{inputDrv}), inputNode, queryResolutionChain))
             return std::nullopt;
 
     rewriteDerivation(store, resolved, inputRewrites);
@@ -1242,13 +1228,13 @@ DerivationOutput DerivationOutput::fromJSON(
         keys.insert(key);
 
     auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
-        auto & method_ = getString(valueAt(json, "method"));
-        ContentAddressMethod method = ContentAddressMethod::parse(method_);
+        ContentAddressMethod method = ContentAddressMethod::parse(
+            getString(valueAt(json, "method")));
         if (method == ContentAddressMethod::Raw::Text)
             xpSettings.require(Xp::DynamicDerivations);
 
-        auto & hashAlgo_ = getString(valueAt(json, "hashAlgo"));
-        auto hashAlgo = parseHashAlgo(hashAlgo_);
+        auto hashAlgo = parseHashAlgo(
+            getString(valueAt(json, "hashAlgo")));
         return { std::move(method), std::move(hashAlgo) };
     };
 
@@ -1365,7 +1351,8 @@ Derivation Derivation::fromJSON(
     res.name = getString(valueAt(json, "name"));
 
     try {
-        for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) {
+        auto outputs = getObject(valueAt(json, "outputs"));
+        for (auto & [outputName, output] : outputs) {
             res.outputs.insert_or_assign(
                 outputName,
                 DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings));
@@ -1376,7 +1363,8 @@ Derivation Derivation::fromJSON(
     }
 
     try {
-        for (auto & input : getArray(valueAt(json, "inputSrcs")))
+        auto inputSrcs = getArray(valueAt(json, "inputSrcs"));
+        for (auto & input : inputSrcs)
             res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
     } catch (Error & e) {
         e.addTrace({}, "while reading key 'inputSrcs'");
@@ -1389,13 +1377,15 @@ Derivation Derivation::fromJSON(
             auto & json = getObject(_json);
             DerivedPathMap<StringSet>::ChildNode node;
             node.value = getStringSet(valueAt(json, "outputs"));
-            for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) {
+            auto drvs = getObject(valueAt(json, "dynamicOutputs"));
+            for (auto & [outputId, childNode] : drvs) {
                 xpSettings.require(Xp::DynamicDerivations);
                 node.childMap[outputId] = doInput(childNode);
             }
             return node;
         };
-        for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(json, "inputDrvs")))
+        auto drvs = getObject(valueAt(json, "inputDrvs"));
+        for (auto & [inputDrvPath, inputOutputs] : drvs)
             res.inputDrvs.map[store.parseStorePath(inputDrvPath)] =
                 doInput(inputOutputs);
     } catch (Error & e) {

src/libstore/derived-path-map.cc

@@ -55,17 +55,17 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
 namespace nix {
 
 template<>
-bool DerivedPathMap<std::set<std::string>>::ChildNode::operator == (
-    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept = default;
+bool DerivedPathMap<StringSet>::ChildNode::operator == (
+    const DerivedPathMap<StringSet>::ChildNode &) const noexcept = default;
 
 // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
 #if 0
 template<>
-std::strong_ordering DerivedPathMap<std::set<std::string>>::ChildNode::operator <=> (
-    const DerivedPathMap<std::set<std::string>>::ChildNode &) const noexcept = default;
+std::strong_ordering DerivedPathMap<StringSet>::ChildNode::operator <=> (
+    const DerivedPathMap<StringSet>::ChildNode &) const noexcept = default;
 #endif
 
-template struct DerivedPathMap<std::set<std::string>>::ChildNode;
-template struct DerivedPathMap<std::set<std::string>>;
+template struct DerivedPathMap<StringSet>::ChildNode;
+template struct DerivedPathMap<StringSet>;
 
 };

src/libstore/derived-path.cc

@@ -170,7 +170,7 @@ void drvRequireExperiment(
 }
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
-    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<const SingleDerivedPath> drv,
     OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -182,7 +182,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 }
 
 DerivedPath::Built DerivedPath::Built::parse(
-    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<const SingleDerivedPath> drv,
     OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -201,7 +201,7 @@ static SingleDerivedPath parseWithSingle(
     return n == s.npos
         ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s)
         : (SingleDerivedPath) SingleDerivedPath::Built::parse(store,
-            make_ref<SingleDerivedPath>(parseWithSingle(
+            make_ref<const SingleDerivedPath>(parseWithSingle(
                 store,
                 s.substr(0, n),
                 separator,
@@ -234,7 +234,7 @@ static DerivedPath parseWith(
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
         : (DerivedPath) DerivedPath::Built::parse(store,
-            make_ref<SingleDerivedPath>(parseWithSingle(
+            make_ref<const SingleDerivedPath>(parseWithSingle(
                 store,
                 s.substr(0, n),
                 separator,

src/libstore/dummy-store.cc

@@ -1,9 +1,9 @@
-#include "nix/store/store-api.hh"
+#include "nix/store/store-registration.hh"
 #include "nix/util/callback.hh"
 
 namespace nix {
 
-struct DummyStoreConfig : virtual StoreConfig {
+struct DummyStoreConfig : public std::enable_shared_from_this<DummyStoreConfig>, virtual StoreConfig {
     using StoreConfig::StoreConfig;
 
     DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params)
@@ -13,35 +13,36 @@ struct DummyStoreConfig : virtual StoreConfig {
             throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
     }
 
-    const std::string name() override { return "Dummy Store"; }
+    static const std::string name() { return "Dummy Store"; }
 
-    std::string doc() override
+    static std::string doc()
     {
         return
           #include "dummy-store.md"
           ;
     }
 
-    static std::set<std::string> uriSchemes() {
+    static StringSet uriSchemes() {
         return {"dummy"};
     }
+
+    ref<Store> openStore() const override;
 };
 
-struct DummyStore : public virtual DummyStoreConfig, public virtual Store
+struct DummyStore : virtual Store
 {
-    DummyStore(std::string_view scheme, std::string_view authority, const Params & params)
-        : StoreConfig(params)
-        , DummyStoreConfig(scheme, authority, params)
-        , Store(params)
-    { }
+    using Config = DummyStoreConfig;
 
-    DummyStore(const Params & params)
-        : DummyStore("dummy", "", params)
+    ref<const Config> config;
+
+    DummyStore(ref<const Config> config)
+        : Store{*config}
+        , config(config)
     { }
 
     std::string getUri() override
     {
-        return *uriSchemes().begin();
+        return *Config::uriSchemes().begin();
     }
 
     void queryPathInfoUncached(const StorePath & path,
@@ -83,9 +84,16 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     { callback(nullptr); }
 
     virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
-    { unsupported("getFSAccessor"); }
+    {
+        return makeEmptySourceAccessor();
+    }
 };
 
-static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
+ref<Store> DummyStore::Config::openStore() const
+{
+    return make_ref<DummyStore>(ref{shared_from_this()});
+}
+
+static RegisterStoreImplementation<DummyStore::Config> regDummyStore;
 
 }

src/libstore/filetransfer.cc

@@ -33,6 +33,9 @@ using namespace std::string_literals;
 
 namespace nix {
 
+const unsigned int RETRY_TIME_MS_DEFAULT = 250;
+const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000;
+
 FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
@@ -309,6 +312,23 @@ struct curlFileTransfer : public FileTransfer
         }
         #endif
 
+        size_t seekCallback(curl_off_t offset, int origin)
+        {
+            if (origin == SEEK_SET) {
+                readOffset = offset;
+            } else if (origin == SEEK_CUR) {
+                readOffset += offset;
+            } else if (origin == SEEK_END) {
+                readOffset = request.data->length() + offset;
+            }
+            return CURL_SEEKFUNC_OK;
+        }
+
+        static size_t seekCallbackWrapper(void *clientp, curl_off_t offset, int origin)
+        {
+            return ((TransferItem *) clientp)->seekCallback(offset, origin);
+        }
+
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -363,6 +383,8 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
                 curl_easy_setopt(req, CURLOPT_READDATA, this);
                 curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+                curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper);
+                curl_easy_setopt(req, CURLOPT_SEEKDATA, this);
             }
 
             if (request.verifyTLS) {
@@ -401,6 +423,8 @@ struct curlFileTransfer : public FileTransfer
         {
             auto finishTime = std::chrono::steady_clock::now();
 
+            auto retryTimeMs = request.baseRetryTimeMs;
+
             auto httpStatus = getHTTPStatus();
 
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
@@ -451,10 +475,12 @@ struct curlFileTransfer : public FileTransfer
                 } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
                     // Don't retry on authentication/authorization failures
                     err = Forbidden;
-                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) {
+                } else if (httpStatus == 429) {
+                    // 429 means too many requests, so we retry (with a substantially longer delay)
+                    retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS;
+                } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) {
                     // Most 4xx errors are client errors and are probably not worth retrying:
                     //   * 408 means the server timed out waiting for us, so we try again
-                    //   * 429 means too many requests, so we retry (with a delay)
                     err = Misc;
                 } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) {
                     // Let's treat most 5xx (server) errors as transient, except for a handful:
@@ -520,7 +546,7 @@ struct curlFileTransfer : public FileTransfer
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
+                    int ms = retryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
@@ -747,7 +773,7 @@ struct curlFileTransfer : public FileTransfer
     }
 
 #if NIX_WITH_S3_SUPPORT
-    std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
+    std::tuple<std::string, std::string, Store::Config::Params> parseS3Uri(std::string uri)
     {
         auto [path, params] = splitUriAndParams(uri);
 

src/libstore/gc.cc

@@ -43,7 +43,7 @@ static std::string gcRootsDir = "gcroots";
 void LocalStore::addIndirectRoot(const Path & path)
 {
     std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
-    Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
+    Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash));
     makeSymlink(realRoot, path);
 }
 
@@ -82,7 +82,7 @@ void LocalStore::createTempRootsFile()
 
 void LocalStore::addTempRoot(const StorePath & path)
 {
-    if (readOnly) {
+    if (config->readOnly) {
       debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
       return;
     }
@@ -109,7 +109,7 @@ void LocalStore::addTempRoot(const StorePath & path)
         auto fdRootsSocket(_fdRootsSocket.lock());
 
         if (!*fdRootsSocket) {
-            auto socketPath = stateDir.get() + gcSocketPath;
+            auto socketPath = config->stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
             *fdRootsSocket = createUnixDomainSocket();
             try {
@@ -164,7 +164,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
-    for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) {
+    for (auto & i : DirectoryIterator{tempRootsDir}) {
         checkInterrupt();
         auto name = i.path().filename().string();
         if (name[0] == '.') {
@@ -232,7 +232,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
             type = std::filesystem::symlink_status(path).type();
 
         if (type == std::filesystem::file_type::directory) {
-            for (auto & i : std::filesystem::directory_iterator{path}) {
+            for (auto & i : DirectoryIterator{path}) {
                 checkInterrupt();
                 findRoots(i.path().string(), i.symlink_status().type(), roots);
             }
@@ -247,7 +247,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
             else {
                 target = absPath(target, dirOf(path));
                 if (!pathExists(target)) {
-                    if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
+                    if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) {
                         printInfo("removing stale link from '%1%' to '%2%'", path, target);
                         unlink(path.c_str());
                     }
@@ -288,8 +288,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
 void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
 {
     /* Process direct roots in {gcroots,profiles}. */
-    findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots);
-    findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots);
+    findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots);
+    findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots);
 
     /* Add additional roots returned by different platforms-specific
        heuristics.  This is typically used to add running programs to
@@ -498,7 +498,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         readFile(*p);
 
     /* Start the server for receiving new roots. */
-    auto socketPath = stateDir.get() + gcSocketPath;
+    auto socketPath = config->stateDir.get() + gcSocketPath;
     createDirs(dirOf(socketPath));
     auto fdServer = createUnixDomainSocket(socketPath, 0666);
 
@@ -635,7 +635,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     auto deleteFromStore = [&](std::string_view baseName)
     {
         Path path = storeDir + "/" + std::string(baseName);
-        Path realPath = realStoreDir + "/" + std::string(baseName);
+        Path realPath = config->realStoreDir + "/" + std::string(baseName);
 
         /* There may be temp directories in the store that are still in use
            by another process. We need to be sure that we can acquire an
@@ -804,8 +804,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             printInfo("determining live/dead paths...");
 
         try {
-            AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
-            if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
+            AutoCloseDir dir(opendir(config->realStoreDir.get().c_str()));
+            if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir);
 
             /* Read the store and delete all paths that are invalid or
                unreachable. We don't use readDirectory() here so that
@@ -907,8 +907,8 @@ void LocalStore::autoGC(bool sync)
             return std::stoll(readFile(*fakeFreeSpaceFile));
 
         struct statvfs st;
-        if (statvfs(realStoreDir.get().c_str(), &st))
-            throw SysError("getting filesystem info about '%s'", realStoreDir);
+        if (statvfs(config->realStoreDir.get().c_str(), &st))
+            throw SysError("getting filesystem info about '%s'", config->realStoreDir);
 
         return (uint64_t) st.f_bavail * st.f_frsize;
     };

src/libstore/http-binary-cache-store.cc

@@ -3,12 +3,22 @@
 #include "nix/store/globals.hh"
 #include "nix/store/nar-info-disk-cache.hh"
 #include "nix/util/callback.hh"
+#include "nix/store/store-registration.hh"
 
 namespace nix {
 
 MakeError(UploadToHTTP, Error);
 
 
+StringSet HttpBinaryCacheStoreConfig::uriSchemes()
+{
+    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
+    auto ret = StringSet{"http", "https"};
+    if (forceHttp)
+        ret.insert("file");
+    return ret;
+}
+
 HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(
     std::string_view scheme,
     std::string_view _cacheUri,
@@ -35,10 +45,9 @@ std::string HttpBinaryCacheStoreConfig::doc()
 }
 
 
-class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
+class HttpBinaryCacheStore :
+    public virtual BinaryCacheStore
 {
-private:
-
     struct State
     {
         bool enabled = true;
@@ -49,37 +58,37 @@ private:
 
 public:
 
-    HttpBinaryCacheStore(
-        std::string_view scheme,
-        PathView cacheUri,
-        const Params & params)
-        : StoreConfig(params)
-        , BinaryCacheStoreConfig(params)
-        , HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
-        , Store(params)
-        , BinaryCacheStore(params)
+    using Config = HttpBinaryCacheStoreConfig;
+
+    ref<Config> config;
+
+    HttpBinaryCacheStore(ref<Config> config)
+        : Store{*config}
+        // TODO it will actually mutate the configuration
+        , BinaryCacheStore{*config}
+        , config{config}
     {
         diskCache = getNarInfoDiskCache();
     }
 
     std::string getUri() override
     {
-        return cacheUri;
+        return config->cacheUri;
     }
 
     void init() override
     {
         // FIXME: do this lazily?
-        if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
-            wantMassQuery.setDefault(cacheInfo->wantMassQuery);
-            priority.setDefault(cacheInfo->priority);
+        if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) {
+            config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
+            config->priority.setDefault(cacheInfo->priority);
         } else {
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
-                throw Error("'%s' does not appear to be a binary cache", cacheUri);
+                throw Error("'%s' does not appear to be a binary cache", config->cacheUri);
             }
-            diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
+            diskCache->createCache(config->cacheUri, config->storeDir, config->wantMassQuery, config->priority);
         }
     }
 
@@ -137,7 +146,7 @@ protected:
         try {
             getFileTransfer()->upload(req);
         } catch (FileTransferError & e) {
-            throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
+            throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg());
         }
     }
 
@@ -146,7 +155,7 @@ protected:
         return FileTransferRequest(
             hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
             ? path
-            : cacheUri + "/" + path);
+            : config->cacheUri + "/" + path);
 
     }
 
@@ -221,6 +230,14 @@ protected:
     }
 };
 
-static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
+ref<Store> HttpBinaryCacheStore::Config::openStore() const
+{
+    return make_ref<HttpBinaryCacheStore>(ref{
+        // FIXME we shouldn't actually need a mutable config
+        std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())
+    });
+}
+
+static RegisterStoreImplementation<HttpBinaryCacheStore::Config> regHttpBinaryCacheStore;
 
 }