nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-30 14:10:59 +01:00
Code Activity

Merge remote-tracking branch 'origin/master' into lazy-trees

Browse source
This commit is contained in:
Eelco Dolstra 2022-08-25 17:41:12 +02:00
commit 13c0db4d06
31 changed files with 282 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/libcmd/repl.cc
View file

@ -35,6 +35,7 @@ extern "C" {
#include "finally.hh"
#include "markdown.hh"
#include "local-fs-store.hh"
#include "progress-bar.hh"
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
@ -250,6 +251,10 @@ void NixRepl::mainLoop()
rl_set_list_possib_func(listPossibleCallback);
#endif
/* Stop the progress bar because it interferes with the display of
the repl. */
stopProgressBar();
std::string input;
while (true) {
@ -1037,9 +1042,10 @@ void runRepl(
struct CmdRepl : InstallablesCommand
{
CmdRepl(){
CmdRepl() {
evalSettings.pureEval = false;
}
void prepare()
{
if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) {
@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand
}
installables = InstallablesCommand::load();
}
std::vector<std::string> files;
Strings getDefaultFlakeAttrPaths() override
{
return {""};
}
virtual bool useDefaultInstallables() override
bool useDefaultInstallables() override
{
return file.has_value() or expr.has_value();
}

5
src/libexpr/eval-cache.cc
View file

@ -510,11 +510,6 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
for (auto & attr : *v.attrs) {
if (root->db)
root->db->setPlaceholder({cachedValue->first, attr.name});
}
auto attr = v.attrs->get(name);
if (!attr) {

27
src/libexpr/value-to-json.cc
View file

@ -11,7 +11,7 @@
namespace nix {
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context)
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore)
{
checkInterrupt();
@ -33,10 +33,13 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
case nPath:
// FIXME: handle accessors
out.write(
state.store->printStorePath(
state.copyPathToStore(context, v.path())));
if (copyToStore)
// FIXME: handle accessors
out.write(
state.store->printStorePath(
state.copyPathToStore(context, v.path())));
else
out.write(v.path().path.abs());
break;
case nNull:
@ -58,10 +61,10 @@ void printValueAsJSON(EvalState & state, bool strict,
for (auto & j : names) {
Attr & a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j));
printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context);
printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore);
}
} else
printValueAsJSON(state, strict, *i->value, i->pos, out, context);
printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore);
break;
}
@ -69,13 +72,13 @@ void printValueAsJSON(EvalState & state, bool strict,
auto list(out.list());
for (auto elem : v.listItems()) {
auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *elem, pos, placeholder, context);
printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore);
}
break;
}
case nExternal:
v.external->printValueAsJSON(state, strict, out, context);
v.external->printValueAsJSON(state, strict, out, context, copyToStore);
break;
case nFloat:
@ -95,14 +98,14 @@ void printValueAsJSON(EvalState & state, bool strict,
}
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, PathSet & context)
Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore)
{
JSONPlaceholder out(str);
printValueAsJSON(state, strict, v, pos, out, context);
printValueAsJSON(state, strict, v, pos, out, context, copyToStore);
}
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context) const
JSONPlaceholder & out, PathSet & context, bool copyToStore) const
{
state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
}

4
src/libexpr/value-to-json.hh
View file

@ -11,9 +11,9 @@ namespace nix {
class JSONPlaceholder;
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context);
Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true);
void printValueAsJSON(EvalState & state, bool strict,
Value & v, const PosIdx pos, std::ostream & str, PathSet & context);
Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true);
}

2
src/libexpr/value.hh
View file

@ -100,7 +100,7 @@ class ExternalValueBase
/* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
virtual void printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context) const;
JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const;
/* Print the value as XML. Defaults to unevaluated */
virtual void printValueAsXML(EvalState & state, bool strict, bool location,

7
src/libmain/loggers.cc
View file

@ -30,8 +30,11 @@ Logger * makeDefaultLogger() {
return makeJSONLogger(*makeSimpleLogger(true));
case LogFormat::bar:
return makeProgressBar();
case LogFormat::barWithLogs:
return makeProgressBar(true);
case LogFormat::barWithLogs: {
auto logger = makeProgressBar();
logger->setPrintBuildLogs(true);
return logger;
}
default:
abort();
}

23
src/libmain/progress-bar.cc
View file

@ -81,14 +81,13 @@ private:
std::condition_variable quitCV, updateCV;
bool printBuildLogs;
bool printBuildLogs = false;
bool isTTY;
public:
ProgressBar(bool printBuildLogs, bool isTTY)
: printBuildLogs(printBuildLogs)
, isTTY(isTTY)
ProgressBar(bool isTTY)
: isTTY(isTTY)
{
state_.lock()->active = isTTY;
updateThread = std::thread([&]() {
@ -503,19 +502,21 @@ public:
draw(*state);
return s[0];
}
virtual void setPrintBuildLogs(bool printBuildLogs)
{
this->printBuildLogs = printBuildLogs;
}
};
Logger * makeProgressBar(bool printBuildLogs)
Logger * makeProgressBar()
{
return new ProgressBar(
printBuildLogs,
shouldANSI()
);
return new ProgressBar(shouldANSI());
}
void startProgressBar(bool printBuildLogs)
void startProgressBar()
{
logger = makeProgressBar(printBuildLogs);
logger = makeProgressBar();
}
void stopProgressBar()

4
src/libmain/progress-bar.hh
View file

@ -4,9 +4,9 @@
namespace nix {
Logger * makeProgressBar(bool printBuildLogs = false);
Logger * makeProgressBar();
void startProgressBar(bool printBuildLogs = false);
void startProgressBar();
void stopProgressBar();

3
src/libstore/filetransfer.cc
View file

@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
if (settings.downloadSpeed.get() > 0)
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
if (request.head)
curl_easy_setopt(req, CURLOPT_NOBODY, 1);

7
src/libstore/globals.hh
View file

@ -746,6 +746,13 @@ public:
/nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
)"};
Setting<unsigned int> downloadSpeed {
this, 0, "download-speed",
R"(
Specify the maximum transfer rate in kilobytes per second you want
Nix to use for downloads.
)"};
Setting<std::string> netrcFile{
this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
R"(

17
src/libstore/remote-store.cc
View file

@ -671,6 +671,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
}
void RemoteStore::addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) {
pathInfo.write(sink, *this, 16);
pathSource->drainInto(sink);
}
});
addMultipleToStore(*source, repair, checkSigs);
}
void RemoteStore::addMultipleToStore(
Source & source,
RepairFlag repair,

6
src/libstore/remote-store.hh
View file

@ -88,6 +88,12 @@ public:
RepairFlag repair,
CheckSigsFlag checkSigs) override;
void addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,

212
src/libstore/store-api.cc
View file

@ -262,6 +262,84 @@ StorePath Store::addToStore(
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
}
void Store::addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0};
std::atomic<uint64_t> bytesExpected{0};
std::atomic<uint64_t> nrRunning{0};
using PathWithInfo = std::pair<ValidPathInfo, std::unique_ptr<Source>>;
std::map<StorePath, PathWithInfo *> infosMap;
StorePathSet storePathsToAdd;
for (auto & thingToAdd : pathsToCopy) {
infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd);
storePathsToAdd.insert(thingToAdd.first.path);
}
auto showProgress = [&]() {
act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
};
ThreadPool pool;
processGraph<StorePath>(pool,
storePathsToAdd,
[&](const StorePath & path) {
auto & [info, _] = *infosMap.at(path);
if (isValidPath(info.path)) {
nrDone++;
showProgress();
return StorePathSet();
}
bytesExpected += info.narSize;
act.setExpected(actCopyPath, bytesExpected);
return info.references;
},
[&](const StorePath & path) {
checkInterrupt();
auto & [info_, source_] = *infosMap.at(path);
auto info = info_;
info.ultimate = false;
/* Make sure that the Source object is destroyed when
we're done. In particular, a SinkToSource object must
be destroyed to ensure that the destructors on its
stack frame are run; this includes
LegacySSHStore::narFromPath()'s connection lock. */
auto source = std::move(source_);
if (!isValidPath(info.path)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
addToStore(info, *source, repair, checkSigs);
} catch (Error & e) {
nrFailed++;
if (!settings.keepGoing)
throw e;
printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what());
showProgress();
return;
}
}
nrDone++;
showProgress();
});
}
void Store::addMultipleToStore(
Source & source,
@ -996,113 +1074,61 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(path);
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
// In the general case, `addMultipleToStore` requires a sorted list of
// store paths to add, so sort them right now
auto sortedMissing = srcStore.topoSortPaths(missing);
std::reverse(sortedMissing.begin(), sortedMissing.end());
std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
Store::PathsSource pathsToCopy;
auto sorted = srcStore.topoSortPaths(missing);
std::reverse(sorted.begin(), sorted.end());
auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath {
auto storePathForSrc = currentPathInfo.path;
auto storePathForDst = storePathForSrc;
if (currentPathInfo.ca && currentPathInfo.references.empty()) {
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca);
if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePathForSrc);
if (storePathForDst != storePathForSrc)
debug("replaced path '%s' to '%s' for substituter '%s'",
srcStore.printStorePath(storePathForSrc),
dstStore.printStorePath(storePathForDst),
dstStore.getUri());
}
return storePathForDst;
};
auto source = sinkToSource([&](Sink & sink) {
sink << sorted.size();
for (auto & storePath : sorted) {
for (auto & missingPath : sortedMissing) {
auto info = srcStore.queryPathInfo(missingPath);
auto storePathForDst = computeStorePathForDst(*info);
pathsMap.insert_or_assign(missingPath, storePathForDst);
ValidPathInfo infoForDst = *info;
infoForDst.path = storePathForDst;
auto source = sinkToSource([&](Sink & sink) {
// We can reasonably assume that the copy will happen whenever we
// read the path, so log something about that at that point
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
auto storePathS = srcStore.printStorePath(storePath);
auto storePathS = srcStore.printStorePath(missingPath);
Activity act(*logger, lvlInfo, actCopyPath,
makeCopyPathMessage(srcUri, dstUri, storePathS),
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
auto info = srcStore.queryPathInfo(storePath);
info->write(sink, srcStore, 16);
srcStore.narFromPath(storePath, sink);
}
});
dstStore.addMultipleToStore(*source, repair, checkSigs);
#if 0
std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0};
std::atomic<uint64_t> bytesExpected{0};
std::atomic<uint64_t> nrRunning{0};
auto showProgress = [&]() {
act.progress(nrDone, missing.size(), nrRunning, nrFailed);
};
ThreadPool pool;
processGraph<StorePath>(pool,
StorePathSet(missing.begin(), missing.end()),
[&](const StorePath & storePath) {
auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'",
srcStore.printStorePath(storePath),
dstStore.printStorePath(storePathForDst),
dstStore.getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (dstStore.isValidPath(storePath)) {
nrDone++;
showProgress();
return StorePathSet();
}
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
return info->references;
},
[&](const StorePath & storePath) {
checkInterrupt();
auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
debug("replaced path '%s' to '%s' for substituter '%s'",
srcStore.printStorePath(storePath),
dstStore.printStorePath(storePathForDst),
dstStore.getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
if (!dstStore.isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
} catch (Error &e) {
nrFailed++;
if (!settings.keepGoing)
throw e;
printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
showProgress();
return;
}
}
nrDone++;
showProgress();
srcStore.narFromPath(missingPath, sink);
});
#endif
pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
}
dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
return pathsMap;
}

11
src/libstore/store-api.hh
View file

@ -1,5 +1,6 @@
#pragma once
#include "nar-info.hh"
#include "realisation.hh"
#include "path.hh"
#include "derived-path.hh"
@ -361,12 +362,22 @@ public:
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
// A list of paths infos along with a source providing the content of the
// associated store path
using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
/* Import multiple paths into the store. */
virtual void addMultipleToStore(
Source & source,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
virtual void addMultipleToStore(
PathsSource & pathsToCopy,
Activity & act,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see

6
src/libutil/json.cc
View file

@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object()
JSONPlaceholder::~JSONPlaceholder()
{
assert(!first || std::uncaught_exceptions());
if (first) {
assert(std::uncaught_exceptions());
if (state->stack != 0)
write(nullptr);
}
}
}

3
src/libutil/logging.hh
View file

@ -111,6 +111,9 @@ public:
virtual std::optional<char> ask(std::string_view s)
{ return {}; }
virtual void setPrintBuildLogs(bool printBuildLogs)
{ }
};
ActivityId getCurActivity();

2
src/nix-build/nix-build.cc
View file

@ -406,7 +406,7 @@ static void main_nix_build(int argc, char * * argv)
auto bashDrv = drv->requireDrvPath();
pathsToBuild.push_back(DerivedPath::Built {
.drvPath = bashDrv,
.outputs = {},
.outputs = {"out"},
});
pathsToCopy.insert(bashDrv);
shellDrv = bashDrv;

4
src/nix-env/nix-env.cc
View file

@ -942,12 +942,12 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
JSONObject metaObj = pkgObj.object("meta");
StringSet metaNames = i.queryMetaNames();
for (auto & j : metaNames) {
auto placeholder = metaObj.placeholder(j);
Value * v = i.queryMeta(j);
if (!v) {
printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
placeholder.write(nullptr);
metaObj.attr(j, nullptr);
} else {
auto placeholder = metaObj.placeholder(j);
PathSet context;
printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context);
}

5
src/nix-instantiate/nix-instantiate.cc
View file

@ -52,9 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths,
state.autoCallFunction(autoArgs, v, vRes);
if (output == okXML)
printValueAsXML(state, strict, location, vRes, std::cout, context, noPos);
else if (output == okJSON)
else if (output == okJSON) {
printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context);
else {
std::cout << std::endl;
} else {
if (strict) state.forceValueDeep(vRes);
vRes.print(state.symbols, std::cout);
std::cout << std::endl;

2
src/nix-store/nix-store.cc
View file

@ -922,7 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs)
if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
if (GET_PROTOCOL_MINOR(clientVersion >= 6)) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 6) {
worker_proto::write(*store, out, status.builtOutputs);
}

3
src/nix/eval.cc
View file

@ -116,7 +116,8 @@ struct CmdEval : MixJSON, InstallableCommand
else if (json) {
JSONPlaceholder jsonOut(std::cout);
printValueAsJSON(*state, true, *v, pos, jsonOut, context);
printValueAsJSON(*state, true, *v, pos, jsonOut, context, false);
std::cout << std::endl;
}
else {

5
src/nix/get-env.sh
View file

@ -43,6 +43,7 @@ __dumpEnv() {
local __var_name="${BASH_REMATCH[2]}"
if [[ $__var_name =~ ^BASH_ || \
$__var_name =~ ^COMP_ || \
$__var_name = _ || \
$__var_name = DIRSTACK || \
$__var_name = EUID || \
@ -54,7 +55,9 @@ __dumpEnv() {
$__var_name = PWD || \
$__var_name = RANDOM || \
$__var_name = SHLVL || \
$__var_name = SECONDS \
$__var_name = SECONDS || \
$__var_name = EPOCHREALTIME || \
$__var_name = EPOCHSECONDS \
]]; then continue; fi
if [[ -z $__first ]]; then printf ',\n'; else __first=; fi

2
src/nix/main.cc
View file

@ -82,7 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
.shortName = 'L',
.description = "Print full build logs on standard error.",
.category = loggingCategory,
.handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
.handler = {[&]() { logger->setPrintBuildLogs(true); }},
});
addFlag({

2
src/nix/profile.md
View file

@ -11,7 +11,7 @@ them to be rolled back easily.
The default profile used by `nix profile` is `$HOME/.nix-profile`,
which, if it does not exist, is created as a symlink to
`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the
`/nix/var/nix/profiles/default` if Nix is invoked by the
`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise.
You can specify another profile location using `--profile` *path*.