Lazily copy trees to the store

We now mount lazy accessors on top of /nix/store without materializing
them, and only materialize them to the real store if needed (e.g. in
the `derivation` primop).

src/libexpr/eval.cc

@@ -267,11 +267,9 @@ EvalState::EvalState(
             auto accessor = getFSSourceAccessor();
 
             auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
-            if (settings.pureEval || store->storeDir != realStoreDir) {
-                accessor = settings.pureEval
-                    ? storeFS.cast<SourceAccessor>()
-                    : makeUnionSourceAccessor({accessor, storeFS});
-            }
+            accessor = settings.pureEval
+                ? storeFS.cast<SourceAccessor>()
+                : makeUnionSourceAccessor({accessor, storeFS});
 
             /* Apply access control if needed. */
             if (settings.restrictEval || settings.pureEval)

src/libexpr/include/nix/expr/eval.hh

@@ -554,6 +554,18 @@ public:
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
         NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
 
+    StorePath devirtualize(
+        const StorePath & path,
+        StringMap * rewrites = nullptr);
+
+    SingleDerivedPath devirtualize(
+        const SingleDerivedPath & path,
+        StringMap * rewrites = nullptr);
+
+    std::string devirtualize(
+        std::string_view s,
+        const NixStringContext & context);
+
     /**
      * String coercion.
      *

src/libexpr/paths.cc

@@ -1,5 +1,7 @@
 #include "nix/store/store-api.hh"
 #include "nix/expr/eval.hh"
+#include "nix/util/mounted-source-accessor.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 namespace nix {
 
@@ -18,4 +20,36 @@ SourcePath EvalState::storePath(const StorePath & path)
     return {rootFS, CanonPath{store->printStorePath(path)}};
 }
 
+StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
+{
+    if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
+        auto storePath = fetchToStore(
+            *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
+        assert(storePath.name() == path.name());
+        if (rewrites)
+            rewrites->emplace(path.hashPart(), storePath.hashPart());
+        return storePath;
+    } else
+        return path;
+}
+
+SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites)
+{
+    if (auto o = std::get_if<SingleDerivedPath::Opaque>(&path.raw()))
+        return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)};
+    else
+        return path;
+}
+
+std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context)
+{
+    StringMap rewrites;
+
+    for (auto & c : context)
+        if (auto o = std::get_if<NixStringContextElem::Opaque>(&c.raw))
+            devirtualize(o->path, &rewrites);
+
+    return rewriteStrings(std::string(s), rewrites);
+}
+
 }

src/libexpr/primops.cc

@@ -14,6 +14,7 @@
 #include "nix/expr/value-to-xml.hh"
 #include "nix/expr/primops.hh"
 #include "nix/fetchers/fetch-to-store.hh"
+#include "nix/util/mounted-source-accessor.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -75,7 +76,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
                 ensureValid(b.drvPath->getBaseStorePath());
             },
             [&](const NixStringContextElem::Opaque & o) {
-                ensureValid(o.path);
+                // We consider virtual store paths valid here. They'll
+                // be devirtualized if needed elsewhere.
+                if (!storeFS->getMount(CanonPath(store->printStorePath(o.path))))
+                    ensureValid(o.path);
                 if (maybePathsOut)
                     maybePathsOut->emplace(o.path);
             },
@@ -1408,6 +1412,8 @@ static void derivationStrictInternal(
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
+    StringMap rewrites;
+
     for (auto & c : context) {
         std::visit(overloaded {
             /* Since this allows the builder to gain access to every
@@ -1430,11 +1436,13 @@ static void derivationStrictInternal(
                 drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
             },
             [&](const NixStringContextElem::Opaque & o) {
-                drv.inputSrcs.insert(o.path);
+                drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
             },
         }, c.raw);
     }
 
+    drv.applyRewrites(rewrites);
+
     /* Do we have all required attributes? */
     if (drv.builder == "")
         state.error<EvalError>("required attribute 'builder' missing")
@@ -2500,6 +2508,7 @@ static void addPath(
                 {}));
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            // FIXME: make this lazy?
             auto dstPath = fetchToStore(
                 *state.store,
                 path.resolveSymlinks(),

src/libexpr/primops/fetchTree.cc

@@ -201,13 +201,16 @@ static void fetchTree(
             throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
     }
 
-    auto [storePath, accessor, input2] = input.fetchToStore(state.store);
+    // FIXME: use fetchOrSubstituteTree().
+    auto [accessor, lockedInput] = input.getAccessor(state.store);
+
+    auto storePath = StorePath::random(input.getName());
 
     state.allowPath(storePath);
 
     state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
 
-    emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
+    emitTreeAttrs(state, storePath, lockedInput, v, params.emptyRevFallback, false);
 }
 
 static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)