diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc
index 887bb47f0..20a4bd6bf 100644
--- a/src/libstore/unix/build/chroot-derivation-builder.cc
+++ b/src/libstore/unix/build/chroot-derivation-builder.cc
@@ -166,9 +166,12 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
         return !needsHashRewrite() ? chrootRootDir + p : store.toRealPath(p);
     }
 
-    void cleanupBuild() override
+    void cleanupBuild(bool force) override
     {
-        DerivationBuilderImpl::cleanupBuild();
+        DerivationBuilderImpl::cleanupBuild(force);
+
+        if (force)
+            return;
 
         /* Move paths out of the chroot for easier debugging of
            build failures. */
diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc
index daaf0b964..bd6cac522 100644
--- a/src/libstore/unix/build/derivation-builder.cc
+++ b/src/libstore/unix/build/derivation-builder.cc
@@ -350,7 +350,7 @@ public:
 
 protected:
 
-    virtual void cleanupBuild();
+    virtual void cleanupBuild(bool force);
 
 private:
 
@@ -480,7 +480,7 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild()
 
         bool diskFull = decideWhetherDiskFull();
 
-        cleanupBuild();
+        cleanupBuild(false);
 
         auto msg =
             fmt("Cannot build '%s'.\n"
@@ -508,14 +508,14 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild()
     for (auto & i : redirectedOutputs)
         deletePath(store.Store::toRealPath(i.second));
 
-    deleteTmpDir(true);
+    cleanupBuild(true);
 
     return builtOutputs;
 }
 
-void DerivationBuilderImpl::cleanupBuild()
+void DerivationBuilderImpl::cleanupBuild(bool force)
 {
-    deleteTmpDir(false);
+    deleteTmpDir(force);
 }
 
 static void chmod_(const Path & path, mode_t mode)