diff --git a/mk/run_test.sh b/mk/run_test.sh index 3783d3bf7..7e95df2ac 100755 --- a/mk/run_test.sh +++ b/mk/run_test.sh @@ -14,9 +14,27 @@ if [ -t 1 ]; then yellow="" normal="" fi -(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) -log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" -status=$? + +run_test () { + (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) + log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" + status=$? +} + +run_test "$1" + +# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these +# appear randomly without anyone knowing why. +# See https://github.com/NixOS/nix/issues/3605 for more info +if [[ $status -ne 0 && $status -ne 99 && \ + "$(uname)" == "Darwin" && \ + "$log" =~ "unexpected EOF reading a line" \ +]]; then + echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)" + echo "$log" | sed 's/^/ /' + run_test "$1" +fi + if [ $status -eq 0 ]; then echo "$post_run_msg [${green}PASS$normal]" elif [ $status -eq 99 ]; then diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index bd8a7ee3a..4bac4b7ba 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -246,7 +246,8 @@ get_volume_pass() { verify_volume_pass() { local volume_special="$1" # (i.e., disk1s7) local volume_uuid="$2" - /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid" + _sudo "to confirm the password actually unlocks the volume" \ + /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid" } volume_pass_works() { @@ -685,22 +686,27 @@ encrypt_volume() { local volume_uuid="$1" local volume_label="$2" local password + + task "Encrypt the Nix volume" >&2 + # Note: mount/unmount are late additions to support the right order # of operations for creating the volume and then baking its uuid into # other artifacts; not as well-trod wrt to potential errors, race # conditions, etc. - /usr/sbin/diskutil mount "$volume_label" + _sudo "to mount your Nix volume for encrypting" \ + /usr/sbin/diskutil mount "$volume_label" password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)" _sudo "to add your Nix volume's password to Keychain" \ /usr/bin/security -i < store) for (auto & p : store->queryAllValidPaths()) paths.push_back(BuiltPath::Opaque{p}); } else { - paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); + paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { // XXX: This only computes the store path closure, ignoring // intermediate realisations diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index bd2a0a7ee..0f6125f11 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -5,7 +5,6 @@ #include "common-eval-args.hh" #include "path.hh" #include "flake/lockfile.hh" -#include "store-api.hh" #include @@ -82,14 +81,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand { return {}; } }; -/* How to handle derivations in commands that operate on store paths. */ -enum class OperateOn { - /* Operate on the output path. */ - Output, - /* Operate on the .drv path. */ - Derivation -}; - struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional file; @@ -113,19 +104,6 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions void completeInstallable(std::string_view prefix); }; -enum class Realise { - /* Build the derivation. Postcondition: the - derivation outputs exist. */ - Outputs, - /* Don't build the derivation. Postcondition: the store derivation - exists. */ - Derivation, - /* Evaluate in dry-run mode. Postcondition: nothing. */ - // FIXME: currently unused, but could be revived if we can - // evaluate derivations in-memory. - Nothing -}; - /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ struct InstallablesCommand : virtual Args, SourceExprCommand @@ -238,38 +216,6 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -BuiltPaths build( - ref evalStore, - ref store, Realise mode, - const std::vector> & installables, - BuildMode bMode = bmNormal); - -std::set toStorePaths( - ref evalStore, - ref store, - Realise mode, - OperateOn operateOn, - const std::vector> & installables); - -StorePath toStorePath( - ref evalStore, - ref store, - Realise mode, - OperateOn operateOn, - std::shared_ptr installable); - -std::set toDerivations( - ref store, - const std::vector> & installables, - bool useDeriver = false); - -BuiltPaths toBuiltPaths( - ref evalStore, - ref store, - Realise mode, - OperateOn operateOn, - const std::vector> & installables); - /* Helper function to generate args that invoke $EDITOR on filename:lineno. */ Strings editorFor(const Pos & pos); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9bce00426..37bfad312 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -468,11 +468,10 @@ std::vector InstallableAttrPath::toDerivations std::vector res; for (auto & drvInfo : drvInfos) { - res.push_back({ - state->store->parseStorePath(drvInfo.queryDrvPath()), - state->store->maybeParseStorePath(drvInfo.queryOutPath()), - drvInfo.queryOutputName() - }); + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + res.push_back({ *drvPath, drvInfo.queryOutputName() }); } return res; @@ -586,9 +585,8 @@ std::tuple InstallableF auto drvPath = attr->forceDerivation(); - auto drvInfo = DerivationInfo{ + auto drvInfo = DerivationInfo { std::move(drvPath), - state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), attr->getAttr(state->sOutputName)->getString() }; @@ -789,7 +787,7 @@ BuiltPaths getBuiltPaths(ref evalStore, ref store, const DerivedPa return res; } -BuiltPaths build( +BuiltPaths Installable::build( ref evalStore, ref store, Realise mode, @@ -814,7 +812,7 @@ BuiltPaths build( return getBuiltPaths(evalStore, store, pathsToBuild); } -BuiltPaths toBuiltPaths( +BuiltPaths Installable::toBuiltPaths( ref evalStore, ref store, Realise mode, @@ -822,19 +820,19 @@ BuiltPaths toBuiltPaths( const std::vector> & installables) { if (operateOn == OperateOn::Output) - return build(evalStore, store, mode, installables); + return Installable::build(evalStore, store, mode, installables); else { if (mode == Realise::Nothing) settings.readOnlyMode = true; BuiltPaths res; - for (auto & drvPath : toDerivations(store, installables, true)) + for (auto & drvPath : Installable::toDerivations(store, installables, true)) res.push_back(BuiltPath::Opaque{drvPath}); return res; } } -StorePathSet toStorePaths( +StorePathSet Installable::toStorePaths( ref evalStore, ref store, Realise mode, OperateOn operateOn, @@ -848,7 +846,7 @@ StorePathSet toStorePaths( return outPaths; } -StorePath toStorePath( +StorePath Installable::toStorePath( ref evalStore, ref store, Realise mode, OperateOn operateOn, @@ -862,7 +860,7 @@ StorePath toStorePath( return *paths.begin(); } -StorePathSet toDerivations( +StorePathSet Installable::toDerivations( ref store, const std::vector> & installables, bool useDeriver) diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 3d2563e4b..e172b71b0 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -5,6 +5,7 @@ #include "path-with-outputs.hh" #include "derived-path.hh" #include "eval.hh" +#include "store-api.hh" #include "flake/flake.hh" #include @@ -29,6 +30,27 @@ struct UnresolvedApp App resolve(ref evalStore, ref store); }; +enum class Realise { + /* Build the derivation. Postcondition: the + derivation outputs exist. */ + Outputs, + /* Don't build the derivation. Postcondition: the store derivation + exists. */ + Derivation, + /* Evaluate in dry-run mode. Postcondition: nothing. */ + // FIXME: currently unused, but could be revived if we can + // evaluate derivations in-memory. + Nothing +}; + +/* How to handle derivations in commands that operate on store paths. */ +enum class OperateOn { + /* Operate on the output path. */ + Output, + /* Operate on the .drv path. */ + Derivation +}; + struct Installable { virtual ~Installable() { } @@ -68,6 +90,39 @@ struct Installable { return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); } + + static BuiltPaths build( + ref evalStore, + ref store, + Realise mode, + const std::vector> & installables, + BuildMode bMode = bmNormal); + + static std::set toStorePaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const std::vector> & installables); + + static StorePath toStorePath( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + std::shared_ptr installable); + + static std::set toDerivations( + ref store, + const std::vector> & installables, + bool useDeriver = false); + + static BuiltPaths toBuiltPaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const std::vector> & installables); }; struct InstallableValue : Installable @@ -79,7 +134,6 @@ struct InstallableValue : Installable struct DerivationInfo { StorePath drvPath; - std::optional outPath; std::string outputName; }; @@ -131,4 +185,9 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake); +BuiltPaths getBuiltPaths( + ref evalStore, + ref store, + const DerivedPaths & hopefullyBuiltPaths); + } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 60f0bf08c..2d7309738 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -86,15 +86,10 @@ RootValue allocRootValue(Value * v) } -void printValue(std::ostream & str, std::set & active, const Value & v) +void printValue(std::ostream & str, std::set & seen, const Value & v) { checkInterrupt(); - if (!active.insert(&v).second) { - str << ""; - return; - } - switch (v.internalType) { case tInt: str << v.integer; @@ -120,24 +115,32 @@ void printValue(std::ostream & str, std::set & active, const Valu str << "null"; break; case tAttrs: { - str << "{ "; - for (auto & i : v.attrs->lexicographicOrder()) { - str << i->name << " = "; - printValue(str, active, *i->value); - str << "; "; + if (!v.attrs->empty() && !seen.insert(v.attrs).second) + str << ""; + else { + str << "{ "; + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, seen, *i->value); + str << "; "; + } + str << "}"; } - str << "}"; break; } case tList1: case tList2: case tListN: - str << "[ "; - for (auto v2 : v.listItems()) { - printValue(str, active, *v2); - str << " "; + if (v.listSize() && !seen.insert(v.listElems()).second) + str << ""; + else { + str << "[ "; + for (auto v2 : v.listItems()) { + printValue(str, seen, *v2); + str << " "; + } + str << "]"; } - str << "]"; break; case tThunk: case tApp: @@ -161,15 +164,13 @@ void printValue(std::ostream & str, std::set & active, const Valu default: abort(); } - - active.erase(&v); } std::ostream & operator << (std::ostream & str, const Value & v) { - std::set active; - printValue(str, active, v); + std::set seen; + printValue(str, seen, v); return str; } @@ -517,6 +518,14 @@ void EvalState::allowPath(const StorePath & storePath) allowedPaths->insert(store->toRealPath(storePath)); } +void EvalState::allowAndSetStorePathString(const StorePath &storePath, Value & v) +{ + allowPath(storePath); + + auto path = store->printStorePath(storePath); + v.mkString(path, PathSet({path})); +} + Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; @@ -2050,6 +2059,18 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) } +StorePath EvalState::coerceToStorePath(const Pos & pos, Value & v, PathSet & context) +{ + auto path = coerceToString(pos, v, context, false, false).toOwned(); + if (auto storePath = store->maybeParseStorePath(path)) + return *storePath; + throw EvalError({ + .msg = hintfmt("path '%1%' is not in the Nix store", path), + .errPos = pos + }); +} + + bool EvalState::eqValues(Value & v1, Value & v2) { forceValue(v1, noPos); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1f0e97b2e..800b00eef 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -161,6 +161,9 @@ public: the real store path if `store` is a chroot store. */ void allowPath(const StorePath & storePath); + /* Allow access to a store path and return it as a string. */ + void allowAndSetStorePathString(const StorePath & storePath, Value & v); + /* Check whether access to a path is allowed and throw an error if not. Otherwise return the canonicalised path. */ Path checkSourcePath(const Path & path); @@ -269,6 +272,9 @@ public: path. Nothing is copied to the store. */ Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + /* Like coerceToPath, but the result must be a store path. */ + StorePath coerceToStorePath(const Pos & pos, Value & v, PathSet & context); + public: /* The base environment, containing the builtin functions and diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 7ecd61816..a811e59a1 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,5 +1,6 @@ #include "flake.hh" #include "globals.hh" +#include "fetch-settings.hh" #include @@ -53,7 +54,7 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - if (nix::settings.acceptFlakeConfig){ + if (nix::fetchSettings.acceptFlakeConfig){ trusted = true; } else if (auto saved = get(get(trustedList, name).value_or(std::map()), valueS)) { trusted = *saved; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 252c1ddc3..8e50f0f22 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,6 +6,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "finally.hh" +#include "fetch-settings.hh" namespace nix { @@ -317,7 +318,7 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); @@ -593,7 +594,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (auto sourcePath = topRef.input.getSourcePath()) { if (!newLockFile.isImmutable()) { - if (settings.warnDirty) + if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); } else { if (!lockFlags.updateLockFile) @@ -620,7 +621,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = settings.commitLockFileSummary.get(); + cm = fetchSettings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); @@ -718,7 +719,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, - .useRegistries = !evalSettings.pureEval && settings.useRegistries, + .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, .allowMutable = !evalSettings.pureEval, }), v); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 930ed9ccd..c1eae413f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -98,7 +98,7 @@ std::pair parseFlakeRefWithFragment( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, - .base = "flake:" + std::string(match[1]), + .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", .path = match[1], @@ -106,12 +106,12 @@ std::pair parseFlakeRefWithFragment( return std::make_pair( FlakeRef(Input::fromURL(parsedURL), ""), - percentDecode(std::string(match[6]))); + percentDecode(match.str(6))); } else if (std::regex_match(url, match, pathUrlRegex)) { std::string path = match[1]; - std::string fragment = percentDecode(std::string(match[3])); + std::string fragment = percentDecode(match.str(3)); if (baseDir) { /* Check if 'url' is a path (either absolute or relative diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 8393e4225..7630c5ff4 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -22,7 +22,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat { auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); - this->drvPath = store->printStorePath(drvPath); + this->drvPath = drvPath; auto drv = store->derivationFromPath(drvPath); @@ -41,9 +41,7 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); auto & [outputName, output] = *i; - auto optStorePath = output.path(*store, drv.name, outputName); - if (optStorePath) - outPath = store->printStorePath(*optStorePath); + outPath = {output.path(*store, drv.name, outputName)}; } @@ -68,24 +66,35 @@ std::string DrvInfo::querySystem() const } -std::string DrvInfo::queryDrvPath() const +std::optional DrvInfo::queryDrvPath() const { - if (drvPath == "" && attrs) { + if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); PathSet context; - drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + if (i == attrs->end()) + drvPath = {std::nullopt}; + else + drvPath = {state->coerceToStorePath(*i->pos, *i->value, context)}; } - return drvPath; + return drvPath.value_or(std::nullopt); } -std::string DrvInfo::queryOutPath() const +StorePath DrvInfo::requireDrvPath() const +{ + if (auto drvPath = queryDrvPath()) + return *drvPath; + throw Error("derivation does not contain a 'drvPath' attribute"); +} + + +StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); PathSet context; if (i != attrs->end()) - outPath = state->coerceToPath(*i->pos, *i->value, context); + outPath = state->coerceToStorePath(*i->pos, *i->value, context); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -113,10 +122,10 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? PathSet context; - outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); + outputs.emplace(name, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); } } else - outputs["out"] = queryOutPath(); + outputs.emplace("out", queryOutPath()); } if (!onlyOutputsToInstall || !attrs) return outputs; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index d13847785..3ca6f1fca 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,6 +1,7 @@ #pragma once #include "eval.hh" +#include "path.hh" #include #include @@ -12,15 +13,15 @@ namespace nix { struct DrvInfo { public: - typedef std::map Outputs; + typedef std::map Outputs; private: EvalState * state; mutable std::string name; mutable std::string system; - mutable std::string drvPath; - mutable std::optional outPath; + mutable std::optional> drvPath; + mutable std::optional outPath; mutable std::string outputName; Outputs outputs; @@ -41,8 +42,9 @@ public: std::string queryName() const; std::string querySystem() const; - std::string queryDrvPath() const; - std::string queryOutPath() const; + std::optional queryDrvPath() const; + StorePath requireDrvPath() const; + StorePath queryOutPath() const; std::string queryOutputName() const; /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); @@ -61,8 +63,8 @@ public: */ void setName(const std::string & s) { name = s; } - void setDrvPath(const std::string & s) { drvPath = s; } - void setOutPath(const std::string & s) { outPath = s; } + void setDrvPath(StorePath path) { drvPath = {{std::move(path)}}; } + void setOutPath(StorePath path) { outPath = {{std::move(path)}}; } void setFailed() { failed = true; }; bool hasFailed() { return failed; }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aa22c3b61..3124025aa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1919,20 +1919,15 @@ static void addPath( if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); - Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - dstPath = state.store->printStorePath(settings.readOnlyMode + StorePath dstPath = settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs)); - if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + if (expectedHash && expectedStorePath != dstPath) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + state.allowAndSetStorePathString(dstPath, v); } else - dstPath = state.store->printStorePath(*expectedStorePath); - - v.mkString(dstPath, {dstPath}); - - state.allowPath(dstPath); - + state.allowAndSetStorePathString(*expectedStorePath, v); } catch (Error & e) { e.addTrace(pos, "while adding path '%s'", path); throw; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f3e3e70d8..9c2da2178 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -230,6 +230,21 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // early exit if pinned and already in the store + if (expectedHash && expectedHash->type == htSHA256) { + auto expectedPath = + unpack + ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) + : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + + if (state.store->isValidPath(expectedPath)) { + state.allowAndSetStorePathString(expectedPath, v); + return; + } + } + + // TODO: fetching may fail, yet the path may be substitutable. + // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath @@ -244,10 +259,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } - state.allowPath(storePath); - - auto path = state.store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + state.allowAndSetStorePathString(storePath, v); } static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 3fdff71a5..d0fa93e92 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -114,8 +114,8 @@ struct Value private: InternalType internalType; -friend std::string showType(const Value & v); -friend void printValue(std::ostream & str, std::set & active, const Value & v); + friend std::string showType(const Value & v); + friend void printValue(std::ostream & str, std::set & seen, const Value & v); public: diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc new file mode 100644 index 000000000..e7d5244dc --- /dev/null +++ b/src/libfetchers/fetch-settings.cc @@ -0,0 +1,13 @@ +#include "fetch-settings.hh" + +namespace nix { + +FetchSettings::FetchSettings() +{ +} + +FetchSettings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh new file mode 100644 index 000000000..58a2aded3 --- /dev/null +++ b/src/libfetchers/fetch-settings.hh @@ -0,0 +1,93 @@ +#pragma once + +#include "types.hh" +#include "config.hh" +#include "util.hh" + +#include +#include + +#include + +namespace nix { + +struct FetchSettings : public Config +{ + FetchSettings(); + + Setting accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type = "gitlab"; + host = "gitlab.mycompany.com"; + owner = "mycompany"; + repo = "pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; + + Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; + + Setting useRegistries{this, true, "use-registries", + "Whether to use flake registries to resolve flake references."}; + + Setting acceptFlakeConfig{this, false, "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting."}; + + Setting commitLockFileSummary{ + this, "", "commit-lockfile-summary", + R"( + The commit summary to use when committing changed flake lock files. If + empty, the summary is generated based on the action performed. + )"}; +}; + +// FIXME: don't use a global variable. +extern FetchSettings fetchSettings; + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7f65c1533..c0beca2f2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,6 +6,8 @@ #include "url-parts.hh" #include "pathlocks.hh" +#include "fetch-settings.hh" + #include #include @@ -246,10 +248,10 @@ struct GitInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!settings.allowDirty) + if (!fetchSettings.allowDirty) throw Error("Git tree '%s' is dirty", actualUrl); - if (settings.warnDirty) + if (fetchSettings.warnDirty) warn("Git tree '%s' is dirty", actualUrl); auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" }); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 70622bf79..a1430f087 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,11 +1,13 @@ #include "filetransfer.hh" #include "cache.hh" -#include "fetchers.hh" #include "globals.hh" #include "store-api.hh" #include "types.hh" #include "url-parts.hh" +#include "fetchers.hh" +#include "fetch-settings.hh" + #include #include #include @@ -157,7 +159,7 @@ struct GitArchiveInputScheme : InputScheme std::optional getAccessToken(const std::string & host) const { - auto tokens = settings.accessTokens.get(); + auto tokens = fetchSettings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 12cdecbc1..8b82e9daa 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -5,6 +5,8 @@ #include "store-api.hh" #include "url-parts.hh" +#include "fetch-settings.hh" + #include using namespace std::string_literals; @@ -165,10 +167,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!settings.allowDirty) + if (!fetchSettings.allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (settings.warnDirty) + if (fetchSettings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index f35359d4b..acd1ff866 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -5,6 +5,8 @@ #include "store-api.hh" #include "local-fs-store.hh" +#include "fetch-settings.hh" + #include namespace nix::fetchers { @@ -150,7 +152,7 @@ void overrideRegistry( static std::shared_ptr getGlobalRegistry(ref store) { static auto reg = [&]() { - auto path = settings.flakeRegistry.get(); + auto path = fetchSettings.flakeRegistry.get(); if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 47fc7ab2e..562d1b414 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,6 +1,7 @@ #include "globals.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" #include "util.hh" #include "loggers.hh" @@ -227,8 +228,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, std::function parseArg) : MixCommonArgs(programName), parseArg(parseArg) { - printError("FOO %s", programName); - addFlag({ .longName = "no-build-output", .shortName = 'Q', diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh new file mode 100644 index 000000000..018ba31b3 --- /dev/null +++ b/src/libstore/build-result.hh @@ -0,0 +1,77 @@ +#pragma once + +#include "realisation.hh" + +#include +#include + + +namespace nix { + +struct BuildResult +{ + /* Note: don't remove status codes, and only add new status codes + at the end of the list, to prevent client/server + incompatibilities in the nix-store --serve protocol. */ + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + CachedFailure, // no longer used + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + ResolvesToAlreadyValid, + } status = MiscFailure; + std::string errorMsg; + + std::string toString() const { + auto strStatus = [&]() { + switch (status) { + case Built: return "Built"; + case Substituted: return "Substituted"; + case AlreadyValid: return "AlreadyValid"; + case PermanentFailure: return "PermanentFailure"; + case InputRejected: return "InputRejected"; + case OutputRejected: return "OutputRejected"; + case TransientFailure: return "TransientFailure"; + case CachedFailure: return "CachedFailure"; + case TimedOut: return "TimedOut"; + case MiscFailure: return "MiscFailure"; + case DependencyFailed: return "DependencyFailed"; + case LogLimitExceeded: return "LogLimitExceeded"; + case NotDeterministic: return "NotDeterministic"; + case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + default: return "Unknown"; + }; + }(); + return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); + } + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + DrvOutputs builtOutputs; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + + bool success() { + return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; + } +}; + +} diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index dfd7ab789..fcb53532e 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,6 +2,7 @@ #include "parsed-derivations.hh" #include "lock.hh" +#include "build-result.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 84a11ee30..20000d4a4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "gc-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -1128,7 +1129,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore { ref next; @@ -1934,7 +1935,7 @@ void LocalDerivationGoal::runChild() "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", i.first, i.second.source); - string path = i.first; + std::string path = i.first; struct stat st; if (lstat(path.c_str(), &st)) { if (i.second.optional && errno == ENOENT) @@ -1986,7 +1987,7 @@ void LocalDerivationGoal::runChild() args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); } args.push_back(drv->builder); } else { @@ -2384,14 +2385,10 @@ void LocalDerivationGoal::registerOutputs() [&](DerivationOutputCAFloating dof) { return newInfoFromCA(dof); }, - [&](DerivationOutputDeferred) { + [&](DerivationOutputDeferred) -> ValidPathInfo { // No derivation should reach that point without having been // rewritten first assert(false); - // Ugly, but the compiler insists on having this return a value - // of type `ValidPathInfo` despite the `assert(false)`, so - // let's provide it - return *(ValidPathInfo*)0; }, }, output.output); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 2d1222d2f..95692c60d 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -62,7 +62,7 @@ struct LocalDerivationGoal : public DerivationGoal Environment env; #if __APPLE__ - typedef string SandboxProfile; + typedef std::string SandboxProfile; SandboxProfile additionalSandboxProfile; #endif diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2ba03f0dd..89d9487da 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,7 +1,9 @@ #include "daemon.hh" #include "monitor-fd.hh" #include "worker-protocol.hh" +#include "build-result.hh" #include "store-api.hh" +#include "gc-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -622,9 +624,12 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddIndirectRoot: { Path path = absPath(readString(from)); + logger->startWork(); - store->addIndirectRoot(path); + auto & gcStore = requireGcStore(*store); + gcStore.addIndirectRoot(path); logger->stopWork(); + to << 1; break; } @@ -639,7 +644,8 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(!trusted); + auto & gcStore = requireGcStore(*store); + Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); size_t size = 0; @@ -670,7 +676,8 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); + auto & gcStore = requireGcStore(*store); + gcStore.collectGarbage(options, results); logger->stopWork(); to << results.paths << results.bytesFreed << 0 /* obsolete */; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 255b42c49..c46262299 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -197,7 +197,7 @@ struct curlFileTransfer : public FileTransfer result.etag = ""; result.data.clear(); result.bodySize = 0; - statusMsg = trim((std::string &) match[1]); + statusMsg = trim(match.str(1)); acceptRanges = false; encoding = ""; } else { diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc new file mode 100644 index 000000000..3dbdec53b --- /dev/null +++ b/src/libstore/gc-store.cc @@ -0,0 +1,13 @@ +#include "gc-store.hh" + +namespace nix { + +GcStore & requireGcStore(Store & store) +{ + auto * gcStore = dynamic_cast(&store); + if (!gcStore) + throw UsageError("Garbage collection not supported by this store"); + return *gcStore; +} + +} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh new file mode 100644 index 000000000..829f70dc4 --- /dev/null +++ b/src/libstore/gc-store.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "store-api.hh" + + +namespace nix { + + +typedef std::unordered_map> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action{gcDeleteDead}; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; + + /* For `gcDeleteSpecific', the paths to delete. */ + StorePathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + uint64_t maxFreed{std::numeric_limits::max()}; +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + uint64_t bytesFreed = 0; +}; + + +struct GcStore : public virtual Store +{ + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) = 0; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) = 0; + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; +}; + +GcStore & requireGcStore(Store & store); + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fd38b731c..024da66c1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -413,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) try { std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = - tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b31a2e8dc..feb6899cd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -880,55 +880,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting accessTokens{this, {}, "access-tokens", - R"( - Access tokens used to access protected GitHub, GitLab, or - other locations requiring token-based authentication. - - Access tokens are specified as a string made up of - space-separated `host=token` values. The specific token - used is selected by matching the `host` portion against the - "host" specification of the input. The actual use of the - `token` value is determined by the type of resource being - accessed: - - * Github: the token value is the OAUTH-TOKEN string obtained - as the Personal Access Token from the Github server (see - https://docs.github.com/en/developers/apps/authorizing-oath-apps). - - * Gitlab: the token value is either the OAuth2 token or the - Personal Access Token (these are different types tokens - for gitlab, see - https://docs.gitlab.com/12.10/ee/api/README.html#authentication). - The `token` value should be `type:tokenstring` where - `type` is either `OAuth2` or `PAT` to indicate which type - of token is being specified. - - Example `~/.config/nix/nix.conf`: - - ``` - access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk - ``` - - Example `~/code/flake.nix`: - - ```nix - input.foo = { - type = "gitlab"; - host = "gitlab.mycompany.com"; - owner = "mycompany"; - repo = "pro"; - }; - ``` - - This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. - - The `input.foo` uses the "gitlab" fetcher, which might - requires specifying the token type along with the token - value. - )"}; - Setting> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; @@ -936,18 +887,9 @@ public: void requireExperimentalFeature(const ExperimentalFeature &); - Setting allowDirty{this, true, "allow-dirty", - "Whether to allow dirty Git/Mercurial trees."}; - - Setting warnDirty{this, true, "warn-dirty", - "Whether to warn about dirty Git/Mercurial trees."}; - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; - Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", - "Path or URI of the global flake registry."}; - Setting allowSymlinkedStore{ this, false, "allow-symlinked-store", R"( @@ -960,19 +902,6 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; - - Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; - - Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting."}; - - Setting commitLockFileSummary{ - this, "", "commit-lockfile-summary", - R"( - The commit summary to use when committing changed flake lock files. If - empty, the summary is generated based on the action performed. - )"}; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6b3daae7f..16c2b90c6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -2,6 +2,7 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" #include "worker-protocol.hh" diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index d34f0cb62..fbd49dc2c 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,6 +1,7 @@ #pragma once #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -23,7 +24,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "physical path to the Nix store"}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore { public: diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1a278c9a8..70d225be3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,6 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" #include "local-fs-store.hh" +#include "gc-store.hh" #include "sync.hh" #include "util.hh" @@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { private: diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 97aa01b57..078c117bd 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const std::vector toDerivedPaths(const std::vector ss) { - std::vector reqs; - for (auto & s : ss) reqs.push_back(s.toDerivedPath()); - return reqs; + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); + return reqs; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e7ffa6595..a25398ef2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,7 +1,9 @@ #include "serialise.hh" #include "util.hh" #include "path-with-outputs.hh" +#include "gc-store.hh" #include "remote-fs-accessor.hh" +#include "build-result.hh" #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b94216d31..2628206b1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,6 +4,7 @@ #include #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, public virtual Store +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore { public: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e4fb1f1fd..7bd21519c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -76,127 +76,10 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; const uint32_t exportMagic = 0x4558494e; -typedef std::unordered_map> Roots; - - -struct GCOptions -{ - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; - - GCAction action{gcDeleteDead}; - - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ - bool ignoreLiveness{false}; - - /* For `gcDeleteSpecific', the paths to delete. */ - StorePathSet pathsToDelete; - - /* Stop after at least `maxFreed' bytes have been freed. */ - uint64_t maxFreed{std::numeric_limits::max()}; -}; - - -struct GCResults -{ - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ - PathSet paths; - - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ - uint64_t bytesFreed = 0; -}; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; +struct BuildResult; -struct BuildResult -{ - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ - enum Status { - Built = 0, - Substituted, - AlreadyValid, - PermanentFailure, - InputRejected, - OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used - TimedOut, - MiscFailure, - DependencyFailed, - LogLimitExceeded, - NotDeterministic, - ResolvesToAlreadyValid, - } status = MiscFailure; - std::string errorMsg; - - std::string toString() const { - auto strStatus = [&]() { - switch (status) { - case Built: return "Built"; - case Substituted: return "Substituted"; - case AlreadyValid: return "AlreadyValid"; - case PermanentFailure: return "PermanentFailure"; - case InputRejected: return "InputRejected"; - case OutputRejected: return "OutputRejected"; - case TransientFailure: return "TransientFailure"; - case CachedFailure: return "CachedFailure"; - case TimedOut: return "TimedOut"; - case MiscFailure: return "MiscFailure"; - case DependencyFailed: return "DependencyFailed"; - case LogLimitExceeded: return "LogLimitExceeded"; - case NotDeterministic: return "NotDeterministic"; - case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; - default: return "Unknown"; - }; - }(); - return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); - } - - /* How many times this build was performed. */ - unsigned int timesBuilt = 0; - - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ - bool isNonDeterministic = false; - - DrvOutputs builtOutputs; - - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ - time_t startTime = 0, stopTime = 0; - - bool success() { - return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; - } -}; struct StoreConfig : public Config { @@ -595,26 +478,6 @@ public: virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) - { unsupported("addIndirectRoot"); } - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ - virtual Roots findRoots(bool censor) - { unsupported("findRoots"); } - - /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) - { unsupported("collectGarbage"); } - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index cef3bd481..2ffdc2e9b 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -359,7 +359,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 88c5ae806..b833038a9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1263,9 +1263,9 @@ std::string chomp(std::string_view s) std::string trim(std::string_view s, std::string_view whitespace) { auto i = s.find_first_not_of(whitespace); - if (i == std::string_view::npos) return ""; + if (i == s.npos) return ""; auto j = s.find_last_not_of(whitespace); - return std::string(s, i, j == std::string::npos ? j : j - i + 1); + return std::string(s, i, j == s.npos ? j : j - i + 1); } @@ -1412,7 +1412,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } } - else if (*i == '\r') + else if (*i == '\r' || *i == '\a') // do nothing for now i++; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3d189f934..795a4f7bd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -346,7 +346,7 @@ static void main_nix_build(int argc, char * * argv) throw UsageError("nix-shell requires a single derivation"); auto & drvInfo = drvs.front(); - auto drv = evalStore->derivationFromPath(evalStore->parseStorePath(drvInfo.queryDrvPath())); + auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath()); std::vector pathsToBuild; RealisedPath::Set pathsToCopy; @@ -369,7 +369,7 @@ static void main_nix_build(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); - auto bashDrv = store->parseStorePath(drv->queryDrvPath()); + auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back({bashDrv}); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -458,10 +458,7 @@ static void main_nix_build(int argc, char * * argv) } } - ParsedDerivation parsedDrv( - StorePath(store->parseStorePath(drvInfo.queryDrvPath())), - drv - ); + ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { auto json = structAttrs.value(); @@ -553,7 +550,7 @@ static void main_nix_build(int argc, char * * argv) std::map> drvMap; for (auto & drvInfo : drvs) { - auto drvPath = store->parseStorePath(drvInfo.queryDrvPath()); + auto drvPath = drvInfo.requireDrvPath(); auto outputName = drvInfo.queryOutputName(); if (outputName == "") diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index f46b90b90..cf52b03b4 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -96,7 +96,7 @@ static void update(const StringSet & channelNames) std::smatch match; auto urlBase = std::string(baseNameOf(url)); if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) - cname = cname + (std::string) match[1]; + cname = cname + match.str(1); std::string extraAttrs; diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 48c030b18..4b28ea6a4 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "gc-store.hh" #include "profiles.hh" #include "shared.hh" #include "globals.hh" @@ -80,10 +81,11 @@ static int main_nix_collect_garbage(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { auto store = openStore(); + auto & gcStore = requireGcStore(*store); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } return 0; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 12e08bbe1..40c3c5d65 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -211,7 +211,7 @@ static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) // at a time. static bool isPrebuilt(EvalState & state, DrvInfo & elem) { - auto path = state.store->parseStorePath(elem.queryOutPath()); + auto path = elem.queryOutPath(); if (state.store->isValidPath(path)) return true; return state.store->querySubstitutablePaths({path}).count(path); } @@ -429,14 +429,15 @@ static void queryInstSources(EvalState & state, elem.setName(name); if (path.isDerivation()) { - elem.setDrvPath(state.store->printStorePath(path)); + elem.setDrvPath(path); auto outputs = state.store->queryDerivationOutputMap(path); - elem.setOutPath(state.store->printStorePath(outputs.at("out"))); + elem.setOutPath(outputs.at("out")); if (name.size() >= drvExtension.size() && std::string(name, name.size() - drvExtension.size()) == drvExtension) name = name.substr(0, name.size() - drvExtension.size()); } - else elem.setOutPath(state.store->printStorePath(path)); + else + elem.setOutPath(path); elems.push_back(elem); } @@ -470,13 +471,11 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { std::vector targets; - for (auto & i : elems) { - Path drvPath = i.queryDrvPath(); - if (drvPath != "") - targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)}); + for (auto & i : elems) + if (auto drvPath = i.queryDrvPath()) + targets.push_back(DerivedPath::Built{*drvPath}); else - targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())}); - } + targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); printMissing(state.store, targets); } @@ -744,14 +743,11 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); + auto drvPath = drv.queryDrvPath(); std::vector paths { - (drv.queryDrvPath() != "") - ? (DerivedPath) (DerivedPath::Built { - globals.state->store->parseStorePath(drv.queryDrvPath()) - }) - : (DerivedPath) (DerivedPath::Opaque { - globals.state->store->parseStorePath(drv.queryOutPath()) - }), + drvPath + ? (DerivedPath) (DerivedPath::Built { *drvPath }) + : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), }; printMissing(globals.state->store, paths); if (globals.dryRun) return; @@ -759,8 +755,9 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) debug(format("switching to new user environment")); Path generation = createGeneration( - ref(store2), globals.profile, - store2->parseStorePath(drv.queryOutPath())); + ref(store2), + globals.profile, + drv.queryOutPath()); switchLink(globals.profile, generation); } @@ -780,7 +777,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, split = std::partition( workingElems.begin(), workingElems.end(), [&selectorStorePath, globals](auto &elem) { - return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath()); + return selectorStorePath != elem.queryOutPath(); } ); } else { @@ -925,9 +922,8 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin if (printOutPath) { DrvInfo::Outputs outputs = i.queryOutputs(); JSONObject outputObj = pkgObj.object("outputs"); - for (auto & j : outputs) { - outputObj.attr(j.first, j.second); - } + for (auto & j : outputs) + outputObj.attr(j.first, globals.state->store->printStorePath(j.second)); } if (printMeta) { @@ -957,6 +953,8 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) { + auto & store { *globals.state->store }; + Strings remaining; std::string attrPath; @@ -1027,12 +1025,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* We only need to know the installed paths when we are querying the status of the derivation. */ - PathSet installed; /* installed paths */ + StorePathSet installed; /* installed paths */ - if (printStatus) { + if (printStatus) for (auto & i : installedElems) installed.insert(i.queryOutPath()); - } /* Query which paths have substitutes. */ @@ -1042,13 +1039,13 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) StorePathSet paths; for (auto & i : elems) try { - paths.insert(globals.state->store->parseStorePath(i.queryOutPath())); + paths.insert(i.queryOutPath()); } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName()); i.setFailed(); } - validPaths = globals.state->store->queryValidPaths(paths); - substitutablePaths = globals.state->store->querySubstitutablePaths(paths); + validPaths = store.queryValidPaths(paths); + substitutablePaths = store.querySubstitutablePaths(paths); } @@ -1073,8 +1070,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); if (globals.prebuiltOnly && - !validPaths.count(globals.state->store->parseStorePath(i.queryOutPath())) && - !substitutablePaths.count(globals.state->store->parseStorePath(i.queryOutPath()))) + !validPaths.count(i.queryOutPath()) && + !substitutablePaths.count(i.queryOutPath())) continue; /* For table output. */ @@ -1084,10 +1081,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) XMLAttrs attrs; if (printStatus) { - Path outPath = i.queryOutPath(); - bool hasSubs = substitutablePaths.count(globals.state->store->parseStorePath(outPath)); - bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.count(globals.state->store->parseStorePath(outPath)); + auto outPath = i.queryOutPath(); + bool hasSubs = substitutablePaths.count(outPath); + bool isInstalled = installed.count(outPath); + bool isValid = validPaths.count(outPath); if (xmlOutput) { attrs["installed"] = isInstalled ? "1" : "0"; attrs["valid"] = isValid ? "1" : "0"; @@ -1152,9 +1149,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (printDrvPath) { auto drvPath = i.queryDrvPath(); if (xmlOutput) { - if (drvPath != "") attrs["drvPath"] = drvPath; + if (drvPath) attrs["drvPath"] = store.printStorePath(*drvPath); } else - columns.push_back(drvPath == "" ? "-" : drvPath); + columns.push_back(drvPath ? store.printStorePath(*drvPath) : "-"); } if (printOutPath && !xmlOutput) { @@ -1163,7 +1160,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto & j : outputs) { if (!s.empty()) s += ';'; if (j.first != "out") { s += j.first; s += "="; } - s += j.second; + s += store.printStorePath(j.second); } columns.push_back(s); } @@ -1184,7 +1181,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto & j : outputs) { XMLAttrs attrs2; attrs2["name"] = j.first; - attrs2["path"] = j.second; + attrs2["path"] = store.printStorePath(j.second); xml.writeEmptyElement("output", attrs2); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 37e4086cb..af4f350ff 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -38,8 +38,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, exist already. */ std::vector drvsToBuild; for (auto & i : elems) - if (i.queryDrvPath() != "") - drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); + if (auto drvPath = i.queryDrvPath()) + drvsToBuild.push_back({*drvPath}); debug(format("building user environment dependencies")); state.store->buildPaths( @@ -55,7 +55,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Create a pseudo-derivation containing the name, system, output paths, and optionally the derivation path, as well as the meta attributes. */ - Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + std::optional drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt; DrvInfo::Outputs outputs = i.queryOutputs(true); StringSet metaNames = i.queryMetaNames(); @@ -66,9 +66,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, auto system = i.querySystem(); if (!system.empty()) attrs.alloc(state.sSystem).mkString(system); - attrs.alloc(state.sOutPath).mkString(i.queryOutPath()); - if (drvPath != "") - attrs.alloc(state.sDrvPath).mkString(i.queryDrvPath()); + attrs.alloc(state.sOutPath).mkString(state.store->printStorePath(i.queryOutPath())); + if (drvPath) + attrs.alloc(state.sDrvPath).mkString(state.store->printStorePath(*drvPath)); // Copy each output meant for installation. auto & vOutputs = attrs.alloc(state.sOutputs); @@ -76,15 +76,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, for (const auto & [m, j] : enumerate(outputs)) { (vOutputs.listElems()[m] = state.allocValue())->mkString(j.first); auto outputAttrs = state.buildBindings(2); - outputAttrs.alloc(state.sOutPath).mkString(j.second); + outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(j.second)); attrs.alloc(j.first).mkAttrs(outputAttrs); /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(state.store->parseStorePath(j.second)); - state.store->ensurePath(state.store->parseStorePath(j.second)); + state.store->addTempRoot(j.second); + state.store->ensurePath(j.second); - references.insert(state.store->parseStorePath(j.second)); + references.insert(j.second); } // Copy the meta attributes. @@ -99,7 +99,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, (manifest.listElems()[n++] = state.allocValue())->mkAttrs(attrs); - if (drvPath != "") references.insert(state.store->parseStorePath(drvPath)); + if (drvPath) references.insert(*drvPath); } /* Also write a copy of the list of user environment elements to @@ -132,9 +132,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); PathSet context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(*aDrvPath.pos, *aDrvPath.value, context)); + auto topLevelDrv = state.coerceToStorePath(*aDrvPath.pos, *aDrvPath.value, context); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - Path topLevelOut = state.coerceToPath(*aOutPath.pos, *aOutPath.value, context); + auto topLevelOut = state.coerceToStorePath(*aOutPath.pos, *aOutPath.value, context); /* Realise the resulting store expression. */ debug("building user environment"); @@ -158,8 +158,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, } debug(format("switching to new user environment")); - Path generation = createGeneration(ref(store2), profile, - store2->parseStorePath(topLevelOut)); + Path generation = createGeneration(ref(store2), profile, topLevelOut); switchLink(profile, generation); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 12fc8baf9..3ec0e6e7c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -61,12 +61,13 @@ void processExpr(EvalState & state, const Strings & attrPaths, DrvInfos drvs; getDerivations(state, v, "", autoArgs, drvs, false); for (auto & i : drvs) { - Path drvPath = i.queryDrvPath(); + auto drvPath = i.requireDrvPath(); + auto drvPathS = state.store->printStorePath(drvPath); /* What output do we want? */ std::string outputName = i.queryOutputName(); if (outputName == "") - throw Error("derivation '%1%' lacks an 'outputName' attribute ", drvPath); + throw Error("derivation '%1%' lacks an 'outputName' attribute", drvPathS); if (gcRoot == "") printGCWarning(); @@ -75,9 +76,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast(); if (store2) - drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName); + drvPathS = store2->addPermRoot(drvPath, rootName); } - std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : "")); + std::cout << fmt("%s%s\n", drvPathS, (outputName != "out" ? "!" + outputName : "")); } } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ecd7279e9..8ebaf9387 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,6 +2,8 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" +#include "build-result.hh" +#include "gc-store.hh" #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" @@ -427,11 +429,12 @@ static void opQuery(Strings opFlags, Strings opArgs) store->computeFSClosure( args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - Roots roots = store->findRoots(false); + auto & gcStore = requireGcStore(*store); + Roots roots = gcStore.findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << fmt("%1% -> %2%\n", link, store->printStorePath(target)); + cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); break; } @@ -587,20 +590,22 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); + auto & gcStore = requireGcStore(*store); + if (printRoots) { - Roots roots = store->findRoots(false); + Roots roots = gcStore.findRoots(false); std::set> roots2; // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) roots2.emplace(link, target); for (auto & [link, target] : roots2) - std::cout << link << " -> " << store->printStorePath(target) << "\n"; + std::cout << link << " -> " << gcStore.printStorePath(target) << "\n"; } else { PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); if (options.action != GCOptions::gcDeleteDead) for (auto & i : results.paths) @@ -624,9 +629,11 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); + auto & gcStore = requireGcStore(*store); + GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } diff --git a/src/nix/app.cc b/src/nix/app.cc index e104cc9c1..2563180fb 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -114,7 +114,7 @@ App UnresolvedApp::resolve(ref evalStore, ref store) installableContext.push_back( std::make_shared(store, ctxElt.toDerivedPath())); - auto builtContext = build(evalStore, store, Realise::Outputs, installableContext); + auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); if (!store->isInStore(res.program)) throw Error("app program '%s' is not in the Nix store", res.program); diff --git a/src/nix/build.cc b/src/nix/build.cc index 521ffce49..b4145f6fa 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,7 +52,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile void run(ref store) override { - auto buildables = build( + auto buildables = Installable::build( getEvalStore(), store, dryRun ? Realise::Derivation : Realise::Outputs, installables, buildMode); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 6b891a6ee..7ed558dee 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -97,13 +97,13 @@ struct CmdBundle : InstallableCommand throw Error("the bundler '%s' does not produce a derivation", bundler.what()); PathSet context2; - StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + auto drvPath = evalState->coerceToStorePath(*attr1->pos, *attr1->value, context2); auto attr2 = vRes->attrs->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); + auto outPath = evalState->coerceToStorePath(*attr2->pos, *attr2->value, context2); store->buildPaths({ DerivedPath::Built { drvPath } }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 92e31599a..8af5da9d0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -307,7 +307,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto builtPaths = toStorePaths( + auto builtPaths = Installable::toStorePaths( getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); for (auto & path: builtPaths) { auto from = store->printStorePath(path); @@ -347,7 +347,7 @@ struct Common : InstallableCommand, MixProfile if (path && hasSuffix(path->to_string(), "-env")) return *path; else { - auto drvs = toDerivations(store, {installable}); + auto drvs = Installable::toDerivations(store, {installable}); if (drvs.size() != 1) throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", @@ -511,7 +511,8 @@ struct CmdDevelop : Common, MixEnvironment nixpkgsLockFlags); shell = store->printStorePath( - toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + + "/bin/bash"; } catch (Error &) { ignoreException(); } diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 734c41e0e..0621d662c 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -131,9 +131,9 @@ struct CmdDiffClosures : SourceExprCommand void run(ref store) override { auto before = parseInstallable(store, _before); - auto beforePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, before); + auto beforePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, before); auto after = parseInstallable(store, _after); - auto afterPath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, after); + auto afterPath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, after); printClosureDiff(store, beforePath, afterPath, ""); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 14a235501..144f8f886 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -327,7 +327,7 @@ struct CmdFlakeCheck : FlakeCommand if (!drvInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); // FIXME: check meta attributes - return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); + return drvInfo->queryDrvPath(); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); reportError(e); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 0e8dc4380..a8ff9c78a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -61,6 +61,27 @@ struct ProfileElement { return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); } + + void updateStorePaths(ref evalStore, ref store, Installable & installable) + { + // FIXME: respect meta.outputsToInstall + storePaths.clear(); + for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) { + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + storePaths.insert(bo.path); + }, + [&](const BuiltPath::Built & bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + storePaths.insert(output.second); + } + }, + }, buildable.raw()); + } + } }; struct ProfileManifest @@ -105,7 +126,7 @@ struct ProfileManifest for (auto & drvInfo : drvInfos) { ProfileElement element; - element.storePaths = {state.store->parseStorePath(drvInfo.queryOutPath())}; + element.storePaths = {drvInfo.queryOutPath()}; elements.emplace_back(std::move(element)); } } @@ -232,53 +253,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); for (auto & installable : installables) { - if (auto installable2 = std::dynamic_pointer_cast(installable)) { - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + ProfileElement element; - ProfileElement element; - if (!drv.outPath) - throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); - element.storePaths = {*drv.outPath}; // FIXME + if (auto installable2 = std::dynamic_pointer_cast(installable)) { + // FIXME: make build() return this? + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); element.source = ProfileElementSource{ installable2->flakeRef, resolvedRef, attrPath, }; - - pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); - - manifest.elements.emplace_back(std::move(element)); - } else { - auto buildables = build(getEvalStore(), store, Realise::Outputs, {installable}, bmNormal); - - for (auto & buildable : buildables) { - ProfileElement element; - - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - pathsToBuild.push_back(bo); - element.storePaths.insert(bo.path); - }, - [&](const BuiltPath::Built & bfd) { - // TODO: Why are we querying if we know the output - // names already? Is it just to figure out what the - // default one is? - for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); - element.storePaths.insert(output.second); - } - }, - }, buildable.raw()); - - manifest.elements.emplace_back(std::move(element)); - } } - } - store->buildPaths(pathsToBuild); + element.updateStorePaths(getEvalStore(), store, *installable); + + manifest.elements.push_back(std::move(element)); + } updateProfile(manifest.build(store)); } @@ -373,15 +366,15 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem if (removedCount == 0) { for (auto matcher: matchers) { - if (const size_t* index = std::get_if(&matcher)){ - warn("'%d' is not a valid index in profile", *index); - } else if (const Path* path = std::get_if(&matcher)){ - warn("'%s' does not match any paths in profile", *path); - } else if (const RegexPattern* regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages in profile", regex->pattern); + if (const size_t * index = std::get_if(&matcher)){ + warn("'%d' is not a valid index", *index); + } else if (const Path * path = std::get_if(&matcher)){ + warn("'%s' does not match any paths", *path); + } else if (const RegexPattern * regex = std::get_if(&matcher)){ + warn("'%s' does not match any packages", regex->pattern); } } - warn ("Try `nix profile list` to see the current profile."); + warn ("Use 'nix profile list' to see the current profile."); } updateProfile(newManifest.build(store)); } @@ -407,8 +400,8 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); - // FIXME: code duplication - std::vector pathsToBuild; + std::vector> installables; + std::vector indices; auto upgradedCount = 0; @@ -423,49 +416,53 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); - InstallableFlake installable( + auto installable = std::make_shared( this, getEvalState(), FlakeRef(element.source->originalRef), "", - {element.source->attrPath}, - {}, + Strings{element.source->attrPath}, + Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + auto [attrPath, resolvedRef, drv] = installable->toDerivation(); if (element.source->resolvedRef == resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", element.source->attrPath, element.source->resolvedRef, resolvedRef); - if (!drv.outPath) - throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); - element.storePaths = {*drv.outPath}; // FIXME element.source = ProfileElementSource{ - installable.flakeRef, + installable->flakeRef, resolvedRef, attrPath, }; - pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {drv.outputName}}); + installables.push_back(installable); + indices.push_back(i); } } if (upgradedCount == 0) { for (auto & matcher : matchers) { - if (const size_t* index = std::get_if(&matcher)){ - warn("'%d' is not a valid index in profile", *index); - } else if (const Path* path = std::get_if(&matcher)){ - warn("'%s' does not match any paths in profile", *path); - } else if (const RegexPattern* regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages in profile", regex->pattern); + if (const size_t * index = std::get_if(&matcher)){ + warn("'%d' is not a valid index", *index); + } else if (const Path * path = std::get_if(&matcher)){ + warn("'%s' does not match any paths", *path); + } else if (const RegexPattern * regex = std::get_if(&matcher)){ + warn("'%s' does not match any packages", regex->pattern); } } warn ("Use 'nix profile list' to see the current profile."); } - store->buildPaths(pathsToBuild); + auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + + for (size_t i = 0; i < installables.size(); ++i) { + auto & installable = installables.at(i); + auto & element = manifest.elements[indices.at(i)]; + element.updateStorePaths(getEvalStore(), store, *installable); + } updateProfile(manifest.build(store)); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 731337004..3a51a13e6 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -384,13 +384,12 @@ StorePath NixRepl::getDerivationPath(Value & v) { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); - Path drvPathRaw = drvInfo->queryDrvPath(); - if (drvPathRaw == "") - throw Error("expression did not evaluate to a valid derivation (no drv path)"); - StorePath drvPath = state->store->parseStorePath(drvPathRaw); - if (!state->store->isValidPath(drvPath)) - throw Error("expression did not evaluate to a valid derivation (invalid drv path)"); - return drvPath; + auto drvPath = drvInfo->queryDrvPath(); + if (!drvPath) + throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); + if (!state->store->isValidPath(*drvPath)) + throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath)); + return *drvPath; } @@ -780,8 +779,11 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "«derivation "; Bindings::iterator i = v.attrs->find(state->sDrvPath); PathSet context; - Path drvPath = i != v.attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "???"; - str << drvPath << "»"; + if (i != v.attrs->end()) + str << state->store->printStorePath(state->coerceToStorePath(*i->pos, *i->value, context)); + else + str << "???"; + str << "»"; } else if (maxDepth > 0) { @@ -798,7 +800,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m else printStringValue(str, i.first.c_str()); str << " = "; - if (seen.find(i.second) != seen.end()) + if (seen.count(i.second)) str << "«repeated»"; else try { diff --git a/src/nix/run.cc b/src/nix/run.cc index a67c23bcb..033263c36 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -91,7 +91,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment void run(ref store) override { - auto outPaths = toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); + auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index c614be68d..61a02c9b3 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -40,7 +40,7 @@ struct CmdShowDerivation : InstallablesCommand void run(ref store) override { - auto drvPaths = toDerivations(store, installables, true); + auto drvPaths = Installable::toDerivations(store, installables, true); if (recursive) { StorePathSet closure; diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index e4a3cb554..aa7a8b12f 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" using namespace nix; @@ -32,12 +33,14 @@ struct CmdStoreDelete : StorePathsCommand void run(ref store, std::vector && storePaths) override { + auto & gcStore = requireGcStore(*store); + for (auto & path : storePaths) options.pathsToDelete.insert(path); GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } }; diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index a2d74066e..21718dc0c 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" using namespace nix; @@ -33,10 +34,12 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref store) override { + auto & gcStore = requireGcStore(*store); + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index cc05deda0..1d9ab28ba 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -82,9 +82,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref store) override { auto package = parseInstallable(store, _package); - auto packagePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); + auto packagePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); auto dependency = parseInstallable(store, _dependency); - auto dependencyPath = toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); + auto dependencyPath = Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; @@ -171,12 +171,6 @@ struct CmdWhyDepends : SourceExprCommand node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", pathS); - } else { - logger->cout("%s%s%s%s" ANSI_NORMAL, - firstPad, - node.visited ? "\e[38;5;244m" : "", - firstPad != "" ? treeLast : "", - pathS); } if (node.path == dependencyPath && !all @@ -184,7 +178,7 @@ struct CmdWhyDepends : SourceExprCommand throw BailOut(); if (node.visited) return; - node.visited = true; + if (precise) node.visited = true; /* Sort the references by distance to `dependency` to ensure that the shortest path is printed first. */ @@ -267,6 +261,16 @@ struct CmdWhyDepends : SourceExprCommand if (!all) break; } + if (!precise) { + auto pathS = store->printStorePath(ref.second->path); + logger->cout("%s%s%s%s" ANSI_NORMAL, + firstPad, + ref.second->visited ? "\e[38;5;244m" : "", + last ? treeLast : treeConn, + pathS); + node.visited = true; + } + printNode(*ref.second, tailPad + (last ? treeNull : treeLine), tailPad + (last ? treeNull : treeLine)); @@ -275,6 +279,9 @@ struct CmdWhyDepends : SourceExprCommand RunPager pager; try { + if (!precise) { + logger->cout("%s", store->printStorePath(graph.at(packagePath).path)); + } printNode(graph.at(packagePath), "", ""); } catch (BailOut & ) { } } diff --git a/tests/bash-profile.sh b/tests/bash-profile.sh new file mode 100644 index 000000000..e2e0d1090 --- /dev/null +++ b/tests/bash-profile.sh @@ -0,0 +1,9 @@ +source common.sh + +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh + +user=$(whoami) +rm -rf $TEST_HOME $TEST_ROOT/profile-var +mkdir -p $TEST_HOME +USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" +USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index 13ef47d2d..1f474dde0 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -2,7 +2,7 @@ source common.sh file=build-hook-ca-floating.nix -sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf +enableFeatures "ca-derivations ca-references" CONTENT_ADDRESSED=true diff --git a/tests/ca/build.sh b/tests/ca/build.sh index c8877f87f..92f8b429a 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -37,7 +37,7 @@ testCutoffFor () { } testCutoff () { - # Don't directly build depenentCA, that way we'll make sure we dodn't rely on + # Don't directly build dependentCA, that way we'll make sure we don't rely on # dependent derivations always being already built. #testDerivation dependentCA testCutoffFor transitivelyDependentCA diff --git a/tests/ca/common.sh b/tests/ca/common.sh index c5aa34334..b9d415863 100644 --- a/tests/ca/common.sh +++ b/tests/ca/common.sh @@ -1,5 +1,5 @@ source ../common.sh -sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf +enableFeatures "ca-derivations ca-references" restartDaemon diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh index 74c5d25fd..da9cd8fb4 100644 --- a/tests/ca/duplicate-realisation-in-closure.sh +++ b/tests/ca/duplicate-realisation-in-closure.sh @@ -2,8 +2,6 @@ source ./common.sh requireDaemonNewerThan "2.4pre20210625" -sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf - export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" diff --git a/tests/ca/nix-copy.sh b/tests/ca/nix-copy.sh index 2e0dea2d2..7a8307a4e 100755 --- a/tests/ca/nix-copy.sh +++ b/tests/ca/nix-copy.sh @@ -2,9 +2,6 @@ source common.sh -# Globally enable the ca derivations experimental flag -sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" - export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" diff --git a/tests/ca/nix-run.sh b/tests/ca/nix-run.sh index 81402af10..5f46518e8 100755 --- a/tests/ca/nix-run.sh +++ b/tests/ca/nix-run.sh @@ -2,8 +2,6 @@ source common.sh -sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf - FLAKE_PATH=path:$PWD nix run --no-write-lock-file $FLAKE_PATH#runnable diff --git a/tests/ca/nix-shell.sh b/tests/ca/nix-shell.sh index 7f1a3a73e..1c5a6639f 100755 --- a/tests/ca/nix-shell.sh +++ b/tests/ca/nix-shell.sh @@ -2,8 +2,6 @@ source common.sh -sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf - CONTENT_ADDRESSED=true cd .. source ./nix-shell.sh diff --git a/tests/ca/post-hook.sh b/tests/ca/post-hook.sh index 1c9d4f700..705bde9d4 100755 --- a/tests/ca/post-hook.sh +++ b/tests/ca/post-hook.sh @@ -4,8 +4,6 @@ source common.sh requireDaemonNewerThan "2.4pre20210626" -sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf - export NIX_TESTS_CA_BY_DEFAULT=1 cd .. source ./post-hook.sh diff --git a/tests/ca/recursive.sh b/tests/ca/recursive.sh index 648bf0a91..0354d23b4 100755 --- a/tests/ca/recursive.sh +++ b/tests/ca/recursive.sh @@ -4,8 +4,6 @@ source common.sh requireDaemonNewerThan "2.4pre20210623" -sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf - export NIX_TESTS_CA_BY_DEFAULT=1 cd .. source ./recursive.sh diff --git a/tests/ca/signatures.sh b/tests/ca/signatures.sh index 0c7d974ea..eb18a4130 100644 --- a/tests/ca/signatures.sh +++ b/tests/ca/signatures.sh @@ -1,8 +1,5 @@ source common.sh -# Globally enable the ca derivations experimental flag -sed -i 's/experimental-features = .*/& ca-derivations ca-references/' "$NIX_CONF_DIR/nix.conf" - clearStore clearCache diff --git a/tests/common.sh.in b/tests/common.sh.in index 49068f1c3..8ce28d318 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -87,15 +87,20 @@ startDaemon() { if [[ "$NIX_REMOTE" == daemon ]]; then return fi - # Start the daemon, wait for the socket to appear. !!! - # ‘nix-daemon’ should have an option to fork into the background. + # Start the daemon, wait for the socket to appear. rm -f $NIX_DAEMON_SOCKET_PATH - PATH=$DAEMON_PATH nix daemon & + PATH=$DAEMON_PATH nix-daemon& + pidDaemon=$! for ((i = 0; i < 300; i++)); do - if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi + if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then + DAEMON_STARTED=1 + break; + fi sleep 0.1 done - pidDaemon=$! + if [[ -z ${DAEMON_STARTED+x} ]]; then + fail "Didn’t manage to start the daemon" + fi trap "killDaemon" EXIT export NIX_REMOTE=daemon } @@ -103,10 +108,10 @@ startDaemon() { killDaemon() { kill $pidDaemon for i in {0..100}; do - kill -0 $pidDaemon || break + kill -0 $pidDaemon 2> /dev/null || break sleep 0.1 done - kill -9 $pidDaemon || true + kill -9 $pidDaemon 2> /dev/null || true wait $pidDaemon || true trap "" EXIT } @@ -167,10 +172,15 @@ needLocalStore() { } # Just to make it easy to find which tests should be fixed -buggyNeedLocalStore () { +buggyNeedLocalStore() { needLocalStore } +enableFeatures() { + local features="$1" + sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf +} + set -x if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 3d1685f43..b41d8c4b7 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -9,6 +9,10 @@ outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file: cmp $outPath fetchurl.sh +# Do not re-fetch paths already present. +outPath2=$(nix-build -vvvvv --expr 'import ' --argstr url file:///does-not-exist/must-remain-unused/fetchurl.sh --argstr sha256 $hash --no-out-link) +test "$outPath" == "$outPath2" + # Now using a base-64 hash. clearStore diff --git a/tests/local.mk b/tests/local.mk index 53a9179a3..c3a6aa1fc 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -89,10 +89,11 @@ nix_tests = \ build.sh \ ca/nix-run.sh \ db-migration.sh \ - nix-profile.sh \ + bash-profile.sh \ pass-as-file.sh \ describe-stores.sh \ - store-ping.sh + store-ping.sh \ + nix-profile.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index e2e0d1090..a7a4d4fa2 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -1,9 +1,97 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +clearStore +clearProfiles -user=$(whoami) -rm -rf $TEST_HOME $TEST_ROOT/profile-var -mkdir -p $TEST_HOME -USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" -USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency +enableFeatures "ca-derivations ca-references" +restartDaemon + +# Make a flake. +flake1Dir=$TEST_ROOT/flake1 +mkdir -p $flake1Dir + +cat > $flake1Dir/flake.nix < \$out/bin/hello < $flake1Dir/who +printf 1.0 > $flake1Dir/version +printf false > $flake1Dir/ca.nix + +cp ./config.nix $flake1Dir/ + +# Test upgrading from nix-env. +nix-env -f ./user-envs.nix -i foo-1.0 +nix profile list | grep '0 - - .*-foo-1.0' +nix profile install $flake1Dir -L +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile history +nix profile history | grep "packages.$system.default: ∅ -> 1.0" +nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' + +# Test upgrading a package. +printf NixOS > $flake1Dir/who +printf 2.0 > $flake1Dir/version +nix profile upgrade 1 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] +nix profile history | grep "packages.$system.default: 1.0 -> 2.0" + +# Test 'history', 'diff-closures'. +nix profile diff-closures + +# Test rollback. +nix profile rollback +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] + +# Test uninstall. +[ -e $TEST_HOME/.nix-profile/bin/foo ] +nix profile remove 0 +(! [ -e $TEST_HOME/.nix-profile/bin/foo ]) +nix profile history | grep 'foo: 1.0 -> ∅' +nix profile diff-closures | grep 'Version 3 -> 4' + +# Test installing a non-flake package. +nix profile install --file ./simple.nix '' +[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +nix profile remove 1 +nix profile install $(nix-build --no-out-link ./simple.nix) +[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] + +# Test wipe-history. +nix profile wipe-history +[[ $(nix profile history | grep Version | wc -l) -eq 1 ]] + +# Test upgrade to CA package. +printf true > $flake1Dir/ca.nix +printf 3.0 > $flake1Dir/version +nix profile upgrade 0 +nix profile history | grep "packages.$system.default: 1.0 -> 3.0" + +# Test new install of CA package. +nix profile remove 0 +printf 4.0 > $flake1Dir/version +printf Utrecht > $flake1Dir/who +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] diff --git a/tests/tarball.sh b/tests/tarball.sh index 1301922a5..d5cab879c 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -26,10 +26,14 @@ test_tarball() { nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + # Do not re-fetch paths already present + nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + # Do not re-fetch paths already present + nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 diff --git a/tests/user-envs.sh b/tests/user-envs.sh index aebf6a2a2..430688de1 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -9,7 +9,6 @@ clearProfiles # Query installed: should be empty. test "$(nix-env -p $profiles/test -q '*' | wc -l)" -eq 0 -mkdir -p $TEST_HOME nix-env --switch-profile $profiles/test # Query available: should contain several.