Merge commit '6636202356' into progress-bar

src/libstore/build-result.hh

@@ -0,0 +1,77 @@
+#pragma once
+
+#include "realisation.hh"
+
+#include <string>
+#include <chrono>
+
+
+namespace nix {
+
+struct BuildResult
+{
+    /* Note: don't remove status codes, and only add new status codes
+       at the end of the list, to prevent client/server
+       incompatibilities in the nix-store --serve protocol. */
+    enum Status {
+        Built = 0,
+        Substituted,
+        AlreadyValid,
+        PermanentFailure,
+        InputRejected,
+        OutputRejected,
+        TransientFailure, // possibly transient
+        CachedFailure, // no longer used
+        TimedOut,
+        MiscFailure,
+        DependencyFailed,
+        LogLimitExceeded,
+        NotDeterministic,
+        ResolvesToAlreadyValid,
+    } status = MiscFailure;
+    std::string errorMsg;
+
+    std::string toString() const {
+        auto strStatus = [&]() {
+            switch (status) {
+                case Built: return "Built";
+                case Substituted: return "Substituted";
+                case AlreadyValid: return "AlreadyValid";
+                case PermanentFailure: return "PermanentFailure";
+                case InputRejected: return "InputRejected";
+                case OutputRejected: return "OutputRejected";
+                case TransientFailure: return "TransientFailure";
+                case CachedFailure: return "CachedFailure";
+                case TimedOut: return "TimedOut";
+                case MiscFailure: return "MiscFailure";
+                case DependencyFailed: return "DependencyFailed";
+                case LogLimitExceeded: return "LogLimitExceeded";
+                case NotDeterministic: return "NotDeterministic";
+                case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
+                default: return "Unknown";
+            };
+        }();
+        return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
+    }
+
+    /* How many times this build was performed. */
+    unsigned int timesBuilt = 0;
+
+    /* If timesBuilt > 1, whether some builds did not produce the same
+       result. (Note that 'isNonDeterministic = false' does not mean
+       the build is deterministic, just that we don't have evidence of
+       non-determinism.) */
+    bool isNonDeterministic = false;
+
+    DrvOutputs builtOutputs;
+
+    /* The start/stop times of the build (or one of the rounds, if it
+       was repeated). */
+    time_t startTime = 0, stopTime = 0;
+
+    bool success() {
+        return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
+    }
+};
+
+}

src/libstore/build/derivation-goal.hh

@@ -2,6 +2,7 @@
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
+#include "build-result.hh"
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "goal.hh"

src/libstore/build/local-derivation-goal.cc

@@ -1,4 +1,5 @@
 #include "local-derivation-goal.hh"
+#include "gc-store.hh"
 #include "hook-instance.hh"
 #include "worker.hh"
 #include "builtins.hh"
@@ -1128,7 +1129,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
 /* A wrapper around LocalStore that only allows building/querying of
    paths that are in the input closures of the build or were added via
    recursive Nix calls. */
-struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
 {
     ref<LocalStore> next;
 
@@ -1934,7 +1935,7 @@ void LocalDerivationGoal::runChild()
                             "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
                             i.first, i.second.source);
 
-                    string path = i.first;
+                    std::string path = i.first;
                     struct stat st;
                     if (lstat(path.c_str(), &st)) {
                         if (i.second.optional && errno == ENOENT)
@@ -1986,7 +1987,7 @@ void LocalDerivationGoal::runChild()
                 args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
                 if (allowLocalNetworking) {
                     args.push_back("-D");
-                    args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+                    args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
                 }
                 args.push_back(drv->builder);
             } else {
@@ -2384,14 +2385,10 @@ void LocalDerivationGoal::registerOutputs()
             [&](DerivationOutputCAFloating dof) {
                 return newInfoFromCA(dof);
             },
-                [&](DerivationOutputDeferred) {
+            [&](DerivationOutputDeferred) -> ValidPathInfo {
                 // No derivation should reach that point without having been
                 // rewritten first
                 assert(false);
-                // Ugly, but the compiler insists on having this return a value
-                // of type `ValidPathInfo` despite the `assert(false)`, so
-                // let's provide it
-                return *(ValidPathInfo*)0;
             },
         }, output.output);
 

src/libstore/build/local-derivation-goal.hh

@@ -62,7 +62,7 @@ struct LocalDerivationGoal : public DerivationGoal
     Environment env;
 
 #if __APPLE__
-    typedef string SandboxProfile;
+    typedef std::string SandboxProfile;
     SandboxProfile additionalSandboxProfile;
 #endif
 

src/libstore/daemon.cc

@@ -1,7 +1,9 @@
 #include "daemon.hh"
 #include "monitor-fd.hh"
 #include "worker-protocol.hh"
+#include "build-result.hh"
 #include "store-api.hh"
+#include "gc-store.hh"
 #include "path-with-outputs.hh"
 #include "finally.hh"
 #include "archive.hh"
@@ -622,9 +624,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopAddIndirectRoot: {
         Path path = absPath(readString(from));
+
         logger->startWork();
-        store->addIndirectRoot(path);
+        auto & gcStore = requireGcStore(*store);
+        gcStore.addIndirectRoot(path);
         logger->stopWork();
+
         to << 1;
         break;
     }
@@ -639,7 +644,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopFindRoots: {
         logger->startWork();
-        Roots roots = store->findRoots(!trusted);
+        auto & gcStore = requireGcStore(*store);
+        Roots roots = gcStore.findRoots(!trusted);
         logger->stopWork();
 
         size_t size = 0;
@@ -670,7 +676,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         if (options.ignoreLiveness)
             throw Error("you are not allowed to ignore liveness");
-        store->collectGarbage(options, results);
+        auto & gcStore = requireGcStore(*store);
+        gcStore.collectGarbage(options, results);
         logger->stopWork();
 
         to << results.paths << results.bytesFreed << 0 /* obsolete */;

src/libstore/filetransfer.cc

@@ -197,7 +197,7 @@ struct curlFileTransfer : public FileTransfer
                 result.etag = "";
                 result.data.clear();
                 result.bodySize = 0;
-                statusMsg = trim((std::string &) match[1]);
+                statusMsg = trim(match.str(1));
                 acceptRanges = false;
                 encoding = "";
             } else {

src/libstore/gc-store.cc

@@ -0,0 +1,13 @@
+#include "gc-store.hh"
+
+namespace nix {
+
+GcStore & requireGcStore(Store & store)
+{
+    auto * gcStore = dynamic_cast<GcStore *>(&store);
+    if (!gcStore)
+        throw UsageError("Garbage collection not supported by this store");
+    return *gcStore;
+}
+
+}

src/libstore/gc-store.hh

@@ -0,0 +1,84 @@
+#pragma once
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+
+typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
+
+
+struct GCOptions
+{
+    /* Garbage collector operation:
+
+       - `gcReturnLive': return the set of paths reachable from
+         (i.e. in the closure of) the roots.
+
+       - `gcReturnDead': return the set of paths not reachable from
+         the roots.
+
+       - `gcDeleteDead': actually delete the latter set.
+
+       - `gcDeleteSpecific': delete the paths listed in
+          `pathsToDelete', insofar as they are not reachable.
+    */
+    typedef enum {
+        gcReturnLive,
+        gcReturnDead,
+        gcDeleteDead,
+        gcDeleteSpecific,
+    } GCAction;
+
+    GCAction action{gcDeleteDead};
+
+    /* If `ignoreLiveness' is set, then reachability from the roots is
+       ignored (dangerous!).  However, the paths must still be
+       unreferenced *within* the store (i.e., there can be no other
+       store paths that depend on them). */
+    bool ignoreLiveness{false};
+
+    /* For `gcDeleteSpecific', the paths to delete. */
+    StorePathSet pathsToDelete;
+
+    /* Stop after at least `maxFreed' bytes have been freed. */
+    uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
+};
+
+
+struct GCResults
+{
+    /* Depending on the action, the GC roots, or the paths that would
+       be or have been deleted. */
+    PathSet paths;
+
+    /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
+       number of bytes that would be or was freed. */
+    uint64_t bytesFreed = 0;
+};
+
+
+struct GcStore : public virtual Store
+{
+    /* Add an indirect root, which is merely a symlink to `path' from
+       /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
+       to be a symlink to a store path.  The garbage collector will
+       automatically remove the indirect root when it finds that
+       `path' has disappeared. */
+    virtual void addIndirectRoot(const Path & path) = 0;
+
+    /* Find the roots of the garbage collector.  Each root is a pair
+       (link, storepath) where `link' is the path of the symlink
+       outside of the Nix store that point to `storePath'. If
+       'censor' is true, privacy-sensitive information about roots
+       found in /proc is censored. */
+    virtual Roots findRoots(bool censor) = 0;
+
+    /* Perform a garbage collection. */
+    virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
+};
+
+GcStore & requireGcStore(Store & store);
+
+}

src/libstore/gc.cc

@@ -413,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
         try {
             std::regex lsofRegex(R"(^n(/.*)$)");
             auto lsofLines =
-                tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
+                tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
             for (const auto & line : lsofLines) {
                 std::smatch match;
                 if (std::regex_match(line, match, lsofRegex))

src/libstore/globals.hh

@@ -880,55 +880,6 @@ public:
           are loaded as plugins (non-recursively).
         )"};
 
-    Setting<StringMap> accessTokens{this, {}, "access-tokens",
-        R"(
-          Access tokens used to access protected GitHub, GitLab, or
-          other locations requiring token-based authentication.
-
-          Access tokens are specified as a string made up of
-          space-separated `host=token` values.  The specific token
-          used is selected by matching the `host` portion against the
-          "host" specification of the input. The actual use of the
-          `token` value is determined by the type of resource being
-          accessed:
-
-          * Github: the token value is the OAUTH-TOKEN string obtained
-            as the Personal Access Token from the Github server (see
-            https://docs.github.com/en/developers/apps/authorizing-oath-apps).
-
-          * Gitlab: the token value is either the OAuth2 token or the
-            Personal Access Token (these are different types tokens
-            for gitlab, see
-            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
-            The `token` value should be `type:tokenstring` where
-            `type` is either `OAuth2` or `PAT` to indicate which type
-            of token is being specified.
-
-          Example `~/.config/nix/nix.conf`:
-
-          ```
-          access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
-          ```
-
-          Example `~/code/flake.nix`:
-
-          ```nix
-          input.foo = {
-            type = "gitlab";
-            host = "gitlab.mycompany.com";
-            owner = "mycompany";
-            repo = "pro";
-          };
-          ```
-
-          This example specifies three tokens, one each for accessing
-          github.com, gitlab.mycompany.com, and sourceforge.net.
-
-          The `input.foo` uses the "gitlab" fetcher, which might
-          requires specifying the token type along with the token
-          value.
-          )"};
-
     Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
@@ -936,18 +887,9 @@ public:
 
     void requireExperimentalFeature(const ExperimentalFeature &);
 
-    Setting<bool> allowDirty{this, true, "allow-dirty",
-        "Whether to allow dirty Git/Mercurial trees."};
-
-    Setting<bool> warnDirty{this, true, "warn-dirty",
-        "Whether to warn about dirty Git/Mercurial trees."};
-
     Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
         "Maximum size of NARs before spilling them to disk."};
 
-    Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
-        "Path or URI of the global flake registry."};
-
     Setting<bool> allowSymlinkedStore{
         this, false, "allow-symlinked-store",
         R"(
@@ -960,19 +902,6 @@ public:
           resolves to a different location from that of the build machine. You
           can enable this setting if you are sure you're not going to do that.
         )"};
-
-    Setting<bool> useRegistries{this, true, "use-registries",
-        "Whether to use flake registries to resolve flake references."};
-
-    Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
-        "Whether to accept nix configuration from a flake without prompting."};
-
-    Setting<std::string> commitLockFileSummary{
-        this, "", "commit-lockfile-summary",
-        R"(
-          The commit summary to use when committing changed flake lock files. If
-          empty, the summary is generated based on the action performed.
-        )"};
 };
 
 

src/libstore/legacy-ssh-store.cc

@@ -2,6 +2,7 @@
 #include "pool.hh"
 #include "remote-store.hh"
 #include "serve-protocol.hh"
+#include "build-result.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 #include "worker-protocol.hh"

src/libstore/local-fs-store.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "store-api.hh"
+#include "gc-store.hh"
 
 namespace nix {
 
@@ -23,7 +24,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
         "physical path to the Nix store"};
 };
 
-class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
+class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore
 {
 public:
 

src/libstore/local-store.hh

@@ -5,6 +5,7 @@
 #include "pathlocks.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
+#include "gc-store.hh"
 #include "sync.hh"
 #include "util.hh"
 
@@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 };
 
 
-class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore
+class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
 {
 private:
 

src/libstore/path-with-outputs.cc

@@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const
 
 std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
 {
-	std::vector<DerivedPath> reqs;
-	for (auto & s : ss) reqs.push_back(s.toDerivedPath());
-	return reqs;
+    std::vector<DerivedPath> reqs;
+    for (auto & s : ss) reqs.push_back(s.toDerivedPath());
+    return reqs;
 }
 
 

src/libstore/remote-store.cc

@@ -1,7 +1,9 @@
 #include "serialise.hh"
 #include "util.hh"
 #include "path-with-outputs.hh"
+#include "gc-store.hh"
 #include "remote-fs-accessor.hh"
+#include "build-result.hh"
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"

src/libstore/remote-store.hh

@@ -4,6 +4,7 @@
 #include <string>
 
 #include "store-api.hh"
+#include "gc-store.hh"
 
 
 namespace nix {
@@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public virtual RemoteStoreConfig, public virtual Store
+class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore
 {
 public:
 

src/libstore/store-api.hh

@@ -76,127 +76,10 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
 const uint32_t exportMagic = 0x4558494e;
 
 
-typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
-
-
-struct GCOptions
-{
-    /* Garbage collector operation:
-
-       - `gcReturnLive': return the set of paths reachable from
-         (i.e. in the closure of) the roots.
-
-       - `gcReturnDead': return the set of paths not reachable from
-         the roots.
-
-       - `gcDeleteDead': actually delete the latter set.
-
-       - `gcDeleteSpecific': delete the paths listed in
-          `pathsToDelete', insofar as they are not reachable.
-    */
-    typedef enum {
-        gcReturnLive,
-        gcReturnDead,
-        gcDeleteDead,
-        gcDeleteSpecific,
-    } GCAction;
-
-    GCAction action{gcDeleteDead};
-
-    /* If `ignoreLiveness' is set, then reachability from the roots is
-       ignored (dangerous!).  However, the paths must still be
-       unreferenced *within* the store (i.e., there can be no other
-       store paths that depend on them). */
-    bool ignoreLiveness{false};
-
-    /* For `gcDeleteSpecific', the paths to delete. */
-    StorePathSet pathsToDelete;
-
-    /* Stop after at least `maxFreed' bytes have been freed. */
-    uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
-};
-
-
-struct GCResults
-{
-    /* Depending on the action, the GC roots, or the paths that would
-       be or have been deleted. */
-    PathSet paths;
-
-    /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
-       number of bytes that would be or was freed. */
-    uint64_t bytesFreed = 0;
-};
-
-
 enum BuildMode { bmNormal, bmRepair, bmCheck };
 
+struct BuildResult;
 
-struct BuildResult
-{
-    /* Note: don't remove status codes, and only add new status codes
-       at the end of the list, to prevent client/server
-       incompatibilities in the nix-store --serve protocol. */
-    enum Status {
-        Built = 0,
-        Substituted,
-        AlreadyValid,
-        PermanentFailure,
-        InputRejected,
-        OutputRejected,
-        TransientFailure, // possibly transient
-        CachedFailure, // no longer used
-        TimedOut,
-        MiscFailure,
-        DependencyFailed,
-        LogLimitExceeded,
-        NotDeterministic,
-        ResolvesToAlreadyValid,
-    } status = MiscFailure;
-    std::string errorMsg;
-
-    std::string toString() const {
-        auto strStatus = [&]() {
-            switch (status) {
-                case Built: return "Built";
-                case Substituted: return "Substituted";
-                case AlreadyValid: return "AlreadyValid";
-                case PermanentFailure: return "PermanentFailure";
-                case InputRejected: return "InputRejected";
-                case OutputRejected: return "OutputRejected";
-                case TransientFailure: return "TransientFailure";
-                case CachedFailure: return "CachedFailure";
-                case TimedOut: return "TimedOut";
-                case MiscFailure: return "MiscFailure";
-                case DependencyFailed: return "DependencyFailed";
-                case LogLimitExceeded: return "LogLimitExceeded";
-                case NotDeterministic: return "NotDeterministic";
-                case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
-                default: return "Unknown";
-            };
-        }();
-        return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
-    }
-
-    /* How many times this build was performed. */
-    unsigned int timesBuilt = 0;
-
-    /* If timesBuilt > 1, whether some builds did not produce the same
-       result. (Note that 'isNonDeterministic = false' does not mean
-       the build is deterministic, just that we don't have evidence of
-       non-determinism.) */
-    bool isNonDeterministic = false;
-
-    DrvOutputs builtOutputs;
-
-    /* The start/stop times of the build (or one of the rounds, if it
-       was repeated). */
-    time_t startTime = 0, stopTime = 0;
-
-    bool success() {
-        return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
-    }
-};
 
 struct StoreConfig : public Config
 {
@@ -595,26 +478,6 @@ public:
     virtual void addTempRoot(const StorePath & path)
     { debug("not creating temporary root, store doesn't support GC"); }
 
-    /* Add an indirect root, which is merely a symlink to `path' from
-       /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
-       to be a symlink to a store path.  The garbage collector will
-       automatically remove the indirect root when it finds that
-       `path' has disappeared. */
-    virtual void addIndirectRoot(const Path & path)
-    { unsupported("addIndirectRoot"); }
-
-    /* Find the roots of the garbage collector.  Each root is a pair
-       (link, storepath) where `link' is the path of the symlink
-       outside of the Nix store that point to `storePath'. If
-       'censor' is true, privacy-sensitive information about roots
-       found in /proc is censored. */
-    virtual Roots findRoots(bool censor)
-    { unsupported("findRoots"); }
-
-    /* Perform a garbage collection. */
-    virtual void collectGarbage(const GCOptions & options, GCResults & results)
-    { unsupported("collectGarbage"); }
-
     /* Return a string representing information about the path that
        can be loaded into the database using `nix-store --load-db' or
        `nix-store --register-validity'. */