mirror of
https://github.com/NixOS/nix.git
synced 2025-12-16 14:01:05 +01:00
Merge remote-tracking branch 'upstream/master' into fix-and-ci-static-builds
This commit is contained in:
John Ericson
commit
39de73550d
193 changed files with 3689 additions and 2107 deletions
|
|
@ -11,6 +11,7 @@
|
|||
#include "nar-accessor.hh"
|
||||
#include "json.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
|
|
@ -22,7 +23,8 @@
|
|||
namespace nix {
|
||||
|
||||
BinaryCacheStore::BinaryCacheStore(const Params & params)
|
||||
: Store(params)
|
||||
: BinaryCacheStoreConfig(params)
|
||||
, Store(params)
|
||||
{
|
||||
if (secretKeyFile != "")
|
||||
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
|
||||
|
|
@ -140,17 +142,10 @@ struct FileSource : FdSource
|
|||
}
|
||||
};
|
||||
|
||||
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
|
||||
std::function<ValidPathInfo(HashResult)> mkInfo)
|
||||
{
|
||||
assert(info.narSize);
|
||||
|
||||
if (!repair && isValidPath(info.path)) {
|
||||
// FIXME: copyNAR -> null sink
|
||||
narSource.drain();
|
||||
return;
|
||||
}
|
||||
|
||||
auto [fdTemp, fnTemp] = createTempFile();
|
||||
|
||||
AutoDelete autoDelete(fnTemp);
|
||||
|
|
@ -160,13 +155,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
|
||||
write the compressed NAR to disk), into a HashSink (to get the
|
||||
NAR hash), and into a NarAccessor (to get the NAR listing). */
|
||||
HashSink fileHashSink(htSHA256);
|
||||
HashSink fileHashSink { htSHA256 };
|
||||
std::shared_ptr<FSAccessor> narAccessor;
|
||||
HashSink narHashSink { htSHA256 };
|
||||
{
|
||||
FdSink fileSink(fdTemp.get());
|
||||
TeeSink teeSink(fileSink, fileHashSink);
|
||||
auto compressionSink = makeCompressionSink(compression, teeSink);
|
||||
TeeSource teeSource(narSource, *compressionSink);
|
||||
TeeSink teeSinkCompressed { fileSink, fileHashSink };
|
||||
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
|
||||
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
|
||||
TeeSource teeSource { narSource, teeSinkUncompressed };
|
||||
narAccessor = makeNarAccessor(teeSource);
|
||||
compressionSink->finish();
|
||||
fileSink.flush();
|
||||
|
|
@ -174,9 +171,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
||||
auto info = mkInfo(narHashSink.finish());
|
||||
auto narInfo = make_ref<NarInfo>(info);
|
||||
narInfo->narSize = info.narSize;
|
||||
narInfo->narHash = info.narHash;
|
||||
narInfo->compression = compression;
|
||||
auto [fileHash, fileSize] = fileHashSink.finish();
|
||||
narInfo->fileHash = fileHash;
|
||||
|
|
@ -298,6 +294,41 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
writeNarInfo(narInfo);
|
||||
|
||||
stats.narInfoWrite++;
|
||||
|
||||
return narInfo;
|
||||
}
|
||||
|
||||
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
if (!repair && isValidPath(info.path)) {
|
||||
// FIXME: copyNAR -> null sink
|
||||
narSource.drain();
|
||||
return;
|
||||
}
|
||||
|
||||
addToStoreCommon(narSource, repair, checkSigs, {[&](HashResult nar) {
|
||||
/* FIXME reinstate these, once we can correctly do hash modulo sink as
|
||||
needed. We need to throw here in case we uploaded a corrupted store path. */
|
||||
// assert(info.narHash == nar.first);
|
||||
// assert(info.narSize == nar.second);
|
||||
return info;
|
||||
}});
|
||||
}
|
||||
|
||||
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||
{
|
||||
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
|
||||
unsupported("addToStoreFromDump");
|
||||
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
|
||||
ValidPathInfo info {
|
||||
makeFixedOutputPath(method, nar.first, name),
|
||||
nar.first,
|
||||
};
|
||||
info.narSize = nar.second;
|
||||
return info;
|
||||
})->path;
|
||||
}
|
||||
|
||||
bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
|
||||
|
|
@ -365,50 +396,52 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
|
|||
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
// FIXME: some cut&paste from LocalStore::addToStore().
|
||||
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
|
||||
non-recursive+sha256 so we can just use the default
|
||||
implementation of this method in terms of addToStoreFromDump. */
|
||||
|
||||
/* Read the whole path into memory. This is not a very scalable
|
||||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
std::optional<Hash> h;
|
||||
HashSink sink { hashAlgo };
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
dumpPath(srcPath, sink, filter);
|
||||
h = hashString(hashAlgo, *sink.s);
|
||||
} else {
|
||||
auto s = readFile(srcPath);
|
||||
dumpString(s, sink);
|
||||
h = hashString(hashAlgo, s);
|
||||
readFile(srcPath, sink);
|
||||
}
|
||||
auto h = sink.finish().first;
|
||||
|
||||
ValidPathInfo info {
|
||||
makeFixedOutputPath(method, *h, name),
|
||||
Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
|
||||
};
|
||||
|
||||
auto source = StringSource { *sink.s };
|
||||
addToStore(info, source, repair, CheckSigs);
|
||||
|
||||
return std::move(info.path);
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
dumpPath(srcPath, sink, filter);
|
||||
});
|
||||
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
|
||||
ValidPathInfo info {
|
||||
makeFixedOutputPath(method, h, name),
|
||||
nar.first,
|
||||
};
|
||||
info.narSize = nar.second;
|
||||
info.ca = FixedOutputHash {
|
||||
.method = method,
|
||||
.hash = h,
|
||||
};
|
||||
return info;
|
||||
})->path;
|
||||
}
|
||||
|
||||
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair)
|
||||
{
|
||||
ValidPathInfo info {
|
||||
computeStorePathForText(name, s, references),
|
||||
Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
|
||||
};
|
||||
info.references = references;
|
||||
auto textHash = hashString(htSHA256, s);
|
||||
auto path = makeTextPath(name, textHash, references);
|
||||
|
||||
if (repair || !isValidPath(info.path)) {
|
||||
StringSink sink;
|
||||
dumpString(s, sink);
|
||||
auto source = StringSource { *sink.s };
|
||||
addToStore(info, source, repair, CheckSigs);
|
||||
}
|
||||
if (!repair && isValidPath(path))
|
||||
return path;
|
||||
|
||||
return std::move(info.path);
|
||||
auto source = StringSource { s };
|
||||
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
|
||||
ValidPathInfo info { path, nar.first };
|
||||
info.narSize = nar.second;
|
||||
info.ca = TextHash { textHash };
|
||||
info.references = references;
|
||||
return info;
|
||||
})->path;
|
||||
}
|
||||
|
||||
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
||||
|
|
|
|||
|
|
@ -11,17 +11,21 @@ namespace nix {
|
|||
|
||||
struct NarInfo;
|
||||
|
||||
class BinaryCacheStore : public Store
|
||||
struct BinaryCacheStoreConfig : virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
||||
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
|
||||
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
||||
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
|
||||
const Setting<bool> parallelCompression{this, false, "parallel-compression",
|
||||
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
||||
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
|
||||
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
||||
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
|
||||
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
|
||||
"enable multi-threading compression, available for xz only currently"};
|
||||
};
|
||||
|
||||
class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
|
||||
{
|
||||
|
||||
private:
|
||||
|
||||
|
|
@ -58,7 +62,7 @@ public:
|
|||
|
||||
public:
|
||||
|
||||
virtual void init();
|
||||
virtual void init() override;
|
||||
|
||||
private:
|
||||
|
||||
|
|
@ -68,6 +72,10 @@ private:
|
|||
|
||||
void writeNarInfo(ref<NarInfo> narInfo);
|
||||
|
||||
ref<const ValidPathInfo> addToStoreCommon(
|
||||
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
|
||||
std::function<ValidPathInfo(HashResult)> mkInfo);
|
||||
|
||||
public:
|
||||
|
||||
bool isValidPathUncached(const StorePath & path) override;
|
||||
|
|
@ -81,6 +89,9 @@ public:
|
|||
void addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
PathFilter & filter, RepairFlag repair) override;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -4,11 +4,13 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::string FixedOutputHash::printMethodAlgo() const {
|
||||
std::string FixedOutputHash::printMethodAlgo() const
|
||||
{
|
||||
return makeFileIngestionPrefix(method) + printHashType(hash.type);
|
||||
}
|
||||
|
||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
|
||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m)
|
||||
{
|
||||
switch (m) {
|
||||
case FileIngestionMethod::Flat:
|
||||
return "";
|
||||
|
|
@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
|
|||
+ hash.to_string(Base32, true);
|
||||
}
|
||||
|
||||
std::string renderContentAddress(ContentAddress ca) {
|
||||
std::string renderContentAddress(ContentAddress ca)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHash th) {
|
||||
return "text:" + th.hash.to_string(Base32, true);
|
||||
|
|
@ -37,54 +40,97 @@ std::string renderContentAddress(ContentAddress ca) {
|
|||
}, ca);
|
||||
}
|
||||
|
||||
ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||
auto rest = rawCa;
|
||||
std::string renderContentAddressMethod(ContentAddressMethod cam)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHashMethod &th) {
|
||||
return std::string{"text:"} + printHashType(htSHA256);
|
||||
},
|
||||
[](FixedOutputHashMethod &fshm) {
|
||||
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
|
||||
}
|
||||
}, cam);
|
||||
}
|
||||
|
||||
/*
|
||||
Parses content address strings up to the hash.
|
||||
*/
|
||||
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
|
||||
{
|
||||
std::string_view wholeInput { rest };
|
||||
|
||||
std::string_view prefix;
|
||||
{
|
||||
auto optPrefix = splitPrefixTo(rest, ':');
|
||||
if (!optPrefix)
|
||||
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
|
||||
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput);
|
||||
prefix = *optPrefix;
|
||||
}
|
||||
|
||||
auto parseHashType_ = [&](){
|
||||
auto hashTypeRaw = splitPrefixTo(rest, ':');
|
||||
if (!hashTypeRaw)
|
||||
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
|
||||
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
|
||||
HashType hashType = parseHashType(*hashTypeRaw);
|
||||
return std::move(hashType);
|
||||
};
|
||||
|
||||
// Switch on prefix
|
||||
if (prefix == "text") {
|
||||
// No parsing of the method, "text" only support flat.
|
||||
// No parsing of the ingestion method, "text" only support flat.
|
||||
HashType hashType = parseHashType_();
|
||||
if (hashType != htSHA256)
|
||||
throw Error("text content address hash should use %s, but instead uses %s",
|
||||
printHashType(htSHA256), printHashType(hashType));
|
||||
return TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
|
||||
};
|
||||
return TextHashMethod {};
|
||||
} else if (prefix == "fixed") {
|
||||
// Parse method
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (splitPrefix(rest, "r:"))
|
||||
method = FileIngestionMethod::Recursive;
|
||||
HashType hashType = parseHashType_();
|
||||
return FixedOutputHash {
|
||||
.method = method,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
|
||||
return FixedOutputHashMethod {
|
||||
.fileIngestionMethod = method,
|
||||
.hashType = std::move(hashType),
|
||||
};
|
||||
} else
|
||||
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
|
||||
}
|
||||
|
||||
ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||
auto rest = rawCa;
|
||||
|
||||
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
|
||||
|
||||
return std::visit(
|
||||
overloaded {
|
||||
[&](TextHashMethod thm) {
|
||||
return ContentAddress(TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
|
||||
});
|
||||
},
|
||||
[&](FixedOutputHashMethod fohMethod) {
|
||||
return ContentAddress(FixedOutputHash {
|
||||
.method = fohMethod.fileIngestionMethod,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
|
||||
});
|
||||
},
|
||||
}, caMethod);
|
||||
}
|
||||
|
||||
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
|
||||
{
|
||||
std::string_view asPrefix {std::string{caMethod} + ":"};
|
||||
return parseContentAddressMethodPrefix(asPrefix);
|
||||
}
|
||||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
|
||||
{
|
||||
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
|
||||
};
|
||||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
|
||||
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
|
||||
};
|
||||
|
||||
std::string renderContentAddress(std::optional<ContentAddress> ca) {
|
||||
std::string renderContentAddress(std::optional<ContentAddress> ca)
|
||||
{
|
||||
return ca ? renderContentAddress(*ca) : "";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,4 +55,23 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
|
|||
|
||||
Hash getContentAddressHash(const ContentAddress & ca);
|
||||
|
||||
/*
|
||||
We only have one way to hash text with references, so this is single-value
|
||||
type is only useful in std::variant.
|
||||
*/
|
||||
struct TextHashMethod { };
|
||||
struct FixedOutputHashMethod {
|
||||
FileIngestionMethod fileIngestionMethod;
|
||||
HashType hashType;
|
||||
};
|
||||
|
||||
typedef std::variant<
|
||||
TextHashMethod,
|
||||
FixedOutputHashMethod
|
||||
> ContentAddressMethod;
|
||||
|
||||
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
|
||||
|
||||
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
#include "monitor-fd.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-store.hh"
|
||||
#include "finally.hh"
|
||||
#include "affinity.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -102,17 +101,20 @@ struct TunnelLogger : public Logger
|
|||
|
||||
/* stopWork() means that we're done; stop sending stderr to the
|
||||
client. */
|
||||
void stopWork(bool success = true, const string & msg = "", unsigned int status = 0)
|
||||
void stopWork(const Error * ex = nullptr)
|
||||
{
|
||||
auto state(state_.lock());
|
||||
|
||||
state->canSendStderr = false;
|
||||
|
||||
if (success)
|
||||
if (!ex)
|
||||
to << STDERR_LAST;
|
||||
else {
|
||||
to << STDERR_ERROR << msg;
|
||||
if (status != 0) to << status;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
|
||||
to << STDERR_ERROR << *ex;
|
||||
} else {
|
||||
to << STDERR_ERROR << ex->what() << ex->status;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -232,7 +234,7 @@ struct ClientSettings
|
|||
else if (setSubstituters(settings.extraSubstituters))
|
||||
;
|
||||
else
|
||||
warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
|
||||
debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
|
||||
} catch (UsageError & e) {
|
||||
warn(e.what());
|
||||
}
|
||||
|
|
@ -240,6 +242,23 @@ struct ClientSettings
|
|||
}
|
||||
};
|
||||
|
||||
static void writeValidPathInfo(
|
||||
ref<Store> store,
|
||||
unsigned int clientVersion,
|
||||
Sink & to,
|
||||
std::shared_ptr<const ValidPathInfo> info)
|
||||
{
|
||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
|
||||
<< info->narHash.to_string(Base16, false);
|
||||
worker_proto::write(*store, to, info->references);
|
||||
to << info->registrationTime << info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
|
||||
to << info->ultimate
|
||||
<< info->sigs
|
||||
<< renderContentAddress(info->ca);
|
||||
}
|
||||
}
|
||||
|
||||
static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
|
||||
Source & from, BufferedSink & to, unsigned int op)
|
||||
|
|
@ -256,11 +275,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopQueryValidPaths: {
|
||||
auto paths = readStorePaths<StorePathSet>(*store, from);
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
logger->startWork();
|
||||
auto res = store->queryValidPaths(paths);
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, res);
|
||||
worker_proto::write(*store, to, res);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -276,11 +295,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopQuerySubstitutablePaths: {
|
||||
auto paths = readStorePaths<StorePathSet>(*store, from);
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
logger->startWork();
|
||||
auto res = store->querySubstitutablePaths(paths);
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, res);
|
||||
worker_proto::write(*store, to, res);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -309,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
paths = store->queryValidDerivers(path);
|
||||
else paths = store->queryDerivationOutputs(path);
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, paths);
|
||||
worker_proto::write(*store, to, paths);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -350,54 +369,90 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopAddToStore: {
|
||||
HashType hashAlgo;
|
||||
std::string baseName;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed;
|
||||
uint8_t recursive;
|
||||
std::string hashAlgoRaw;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
hashAlgoRaw = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
|
||||
auto name = readString(from);
|
||||
auto camStr = readString(from);
|
||||
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
bool repairBool;
|
||||
from >> repairBool;
|
||||
auto repair = RepairFlag{repairBool};
|
||||
|
||||
logger->startWork();
|
||||
auto pathInfo = [&]() {
|
||||
// NB: FramedSource must be out of scope before logger->stopWork();
|
||||
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
|
||||
FramedSource source(from);
|
||||
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
||||
return std::visit(overloaded {
|
||||
[&](TextHashMethod &_) {
|
||||
// We could stream this by changing Store
|
||||
std::string contents = source.drain();
|
||||
auto path = store->addTextToStore(name, contents, refs, repair);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
[&](FixedOutputHashMethod &fohm) {
|
||||
if (!refs.empty())
|
||||
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
|
||||
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
}, contentAddressMethod);
|
||||
}();
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(pathInfo->path);
|
||||
writeValidPathInfo(store, clientVersion, to, pathInfo);
|
||||
|
||||
} else {
|
||||
HashType hashAlgo;
|
||||
std::string baseName;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed;
|
||||
uint8_t recursive;
|
||||
std::string hashAlgoRaw;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
hashAlgoRaw = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
}
|
||||
hashAlgo = parseHashType(hashAlgoRaw);
|
||||
}
|
||||
hashAlgo = parseHashType(hashAlgoRaw);
|
||||
|
||||
StringSink saved;
|
||||
TeeSource savedNARSource(from, saved);
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
parseDump(sink, savedNARSource);
|
||||
} else
|
||||
parseDump(savedRegular, from);
|
||||
|
||||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
// FIXME: try to stream directly from `from`.
|
||||
StringSource dumpSource { *saved.s };
|
||||
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
}
|
||||
|
||||
StringSink saved;
|
||||
TeeSource savedNARSource(from, saved);
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
parseDump(sink, savedNARSource);
|
||||
} else
|
||||
parseDump(savedRegular, from);
|
||||
|
||||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
// FIXME: try to stream directly from `from`.
|
||||
StringSource dumpSource { *saved.s };
|
||||
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
break;
|
||||
}
|
||||
|
||||
case wopAddTextToStore: {
|
||||
string suffix = readString(from);
|
||||
string s = readString(from);
|
||||
auto refs = readStorePaths<StorePathSet>(*store, from);
|
||||
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
logger->startWork();
|
||||
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
|
||||
logger->stopWork();
|
||||
|
|
@ -494,6 +549,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
are in fact content-addressed if we don't trust them. */
|
||||
assert(derivationIsCA(drv.type()) || trusted);
|
||||
|
||||
/* Recompute the derivation path when we cannot trust the original. */
|
||||
if (!trusted) {
|
||||
/* Recomputing the derivation path for input-address derivations
|
||||
makes it harder to audit them after the fact, since we need the
|
||||
original not-necessarily-resolved derivation to verify the drv
|
||||
derivation as adequate claim to the input-addressed output
|
||||
paths. */
|
||||
assert(derivationIsCA(drv.type()));
|
||||
|
||||
Derivation drv2;
|
||||
static_cast<BasicDerivation &>(drv2) = drv;
|
||||
drvPath = writeDerivation(*store, Derivation { drv2 });
|
||||
}
|
||||
|
||||
auto res = store->buildDerivation(drvPath, drv, buildMode);
|
||||
logger->stopWork();
|
||||
to << res.status << res.errorMsg;
|
||||
|
|
@ -556,7 +625,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
case wopCollectGarbage: {
|
||||
GCOptions options;
|
||||
options.action = (GCOptions::GCAction) readInt(from);
|
||||
options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
|
||||
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
from >> options.ignoreLiveness >> options.maxFreed;
|
||||
// obsolete fields
|
||||
readInt(from);
|
||||
|
|
@ -625,7 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
else {
|
||||
to << 1
|
||||
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
|
||||
writeStorePaths(*store, to, i->second.references);
|
||||
worker_proto::write(*store, to, i->second.references);
|
||||
to << i->second.downloadSize
|
||||
<< i->second.narSize;
|
||||
}
|
||||
|
|
@ -636,11 +705,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
SubstitutablePathInfos infos;
|
||||
StorePathCAMap pathsMap = {};
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
|
||||
auto paths = readStorePaths<StorePathSet>(*store, from);
|
||||
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
for (auto & path : paths)
|
||||
pathsMap.emplace(path, std::nullopt);
|
||||
} else
|
||||
pathsMap = readStorePathCAMap(*store, from);
|
||||
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
|
||||
logger->startWork();
|
||||
store->querySubstitutablePathInfos(pathsMap, infos);
|
||||
logger->stopWork();
|
||||
|
|
@ -648,7 +717,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
for (auto & i : infos) {
|
||||
to << store->printStorePath(i.first)
|
||||
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
|
||||
writeStorePaths(*store, to, i.second.references);
|
||||
worker_proto::write(*store, to, i.second.references);
|
||||
to << i.second.downloadSize << i.second.narSize;
|
||||
}
|
||||
break;
|
||||
|
|
@ -658,7 +727,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
logger->startWork();
|
||||
auto paths = store->queryAllValidPaths();
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, paths);
|
||||
worker_proto::write(*store, to, paths);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -675,15 +744,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
if (info) {
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
|
||||
to << 1;
|
||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
|
||||
<< info->narHash.to_string(Base16, false);
|
||||
writeStorePaths(*store, to, info->references);
|
||||
to << info->registrationTime << info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
|
||||
to << info->ultimate
|
||||
<< info->sigs
|
||||
<< renderContentAddress(info->ca);
|
||||
}
|
||||
writeValidPathInfo(store, clientVersion, to, info);
|
||||
} else {
|
||||
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
|
||||
to << 0;
|
||||
|
|
@ -738,7 +799,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
ValidPathInfo info { path, narHash };
|
||||
if (deriver != "")
|
||||
info.deriver = store->parseStorePath(deriver);
|
||||
info.references = readStorePaths<StorePathSet>(*store, from);
|
||||
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
||||
from >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||
info.sigs = readStrings<StringSet>(from);
|
||||
info.ca = parseContentAddressOpt(readString(from));
|
||||
|
|
@ -749,59 +810,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
info.ultimate = false;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
|
||||
|
||||
struct FramedSource : Source
|
||||
{
|
||||
Source & from;
|
||||
bool eof = false;
|
||||
std::vector<unsigned char> pending;
|
||||
size_t pos = 0;
|
||||
|
||||
FramedSource(Source & from) : from(from)
|
||||
{ }
|
||||
|
||||
~FramedSource()
|
||||
{
|
||||
if (!eof) {
|
||||
while (true) {
|
||||
auto n = readInt(from);
|
||||
if (!n) break;
|
||||
std::vector<unsigned char> data(n);
|
||||
from(data.data(), n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
if (eof) throw EndOfFile("reached end of FramedSource");
|
||||
|
||||
if (pos >= pending.size()) {
|
||||
size_t len = readInt(from);
|
||||
if (!len) {
|
||||
eof = true;
|
||||
return 0;
|
||||
}
|
||||
pending = std::vector<unsigned char>(len);
|
||||
pos = 0;
|
||||
from(pending.data(), len);
|
||||
}
|
||||
|
||||
auto n = std::min(len, pending.size() - pos);
|
||||
memcpy(data, pending.data() + pos, n);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
logger->startWork();
|
||||
|
||||
{
|
||||
FramedSource source(from);
|
||||
store->addToStore(info, source, (RepairFlag) repair,
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
}
|
||||
|
||||
logger->stopWork();
|
||||
}
|
||||
|
||||
|
|
@ -838,9 +852,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
uint64_t downloadSize, narSize;
|
||||
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, willBuild);
|
||||
writeStorePaths(*store, to, willSubstitute);
|
||||
writeStorePaths(*store, to, unknown);
|
||||
worker_proto::write(*store, to, willBuild);
|
||||
worker_proto::write(*store, to, willSubstitute);
|
||||
worker_proto::write(*store, to, unknown);
|
||||
to << downloadSize << narSize;
|
||||
break;
|
||||
}
|
||||
|
|
@ -924,10 +938,11 @@ void processConnection(
|
|||
during addTextToStore() / importPath(). If that
|
||||
happens, just send the error message and exit. */
|
||||
bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
|
||||
tunnelLogger->stopWork(false, e.msg(), e.status);
|
||||
tunnelLogger->stopWork(&e);
|
||||
if (!errorAllowed) throw;
|
||||
} catch (std::bad_alloc & e) {
|
||||
tunnelLogger->stopWork(false, "Nix daemon out of memory", 1);
|
||||
auto ex = Error("Nix daemon out of memory");
|
||||
tunnelLogger->stopWork(&ex);
|
||||
throw;
|
||||
}
|
||||
|
||||
|
|
@ -936,8 +951,13 @@ void processConnection(
|
|||
assert(!tunnelLogger->state_.lock()->canSendStderr);
|
||||
};
|
||||
|
||||
} catch (Error & e) {
|
||||
tunnelLogger->stopWork(&e);
|
||||
to.flush();
|
||||
return;
|
||||
} catch (std::exception & e) {
|
||||
tunnelLogger->stopWork(false, e.what(), 1);
|
||||
auto ex = Error(e.what());
|
||||
tunnelLogger->stopWork(&ex);
|
||||
to.flush();
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#pragma once
|
||||
|
||||
#include "serialise.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
|
||||
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
|
||||
|
|
@ -15,7 +15,7 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
|
|||
},
|
||||
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
|
||||
return {
|
||||
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
|
||||
dof.path(store, drvName, outputName)
|
||||
};
|
||||
},
|
||||
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
|
||||
|
|
@ -25,6 +25,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
|
|||
}
|
||||
|
||||
|
||||
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
|
||||
return store.makeFixedOutputPath(
|
||||
hash.method, hash.hash,
|
||||
outputPathName(drvName, outputName));
|
||||
}
|
||||
|
||||
|
||||
bool derivationIsCA(DerivationType dt) {
|
||||
switch (dt) {
|
||||
case DerivationType::InputAddressed: return false;
|
||||
|
|
@ -62,7 +69,7 @@ bool BasicDerivation::isBuiltin() const
|
|||
|
||||
|
||||
StorePath writeDerivation(Store & store,
|
||||
const Derivation & drv, RepairFlag repair)
|
||||
const Derivation & drv, RepairFlag repair, bool readOnly)
|
||||
{
|
||||
auto references = drv.inputSrcs;
|
||||
for (auto & i : drv.inputDrvs)
|
||||
|
|
@ -72,7 +79,7 @@ StorePath writeDerivation(Store & store,
|
|||
held during a garbage collection). */
|
||||
auto suffix = std::string(drv.name) + drvExtension;
|
||||
auto contents = drv.unparse(store, false);
|
||||
return settings.readOnlyMode
|
||||
return readOnly || settings.readOnlyMode
|
||||
? store.computeStorePathForText(suffix, contents, references)
|
||||
: store.addTextToStore(suffix, contents, references, repair);
|
||||
}
|
||||
|
|
@ -106,12 +113,15 @@ static string parseString(std::istream & str)
|
|||
return res;
|
||||
}
|
||||
|
||||
static void validatePath(std::string_view s) {
|
||||
if (s.size() == 0 || s[0] != '/')
|
||||
throw FormatError("bad path '%1%' in derivation", s);
|
||||
}
|
||||
|
||||
static Path parsePath(std::istream & str)
|
||||
{
|
||||
string s = parseString(str);
|
||||
if (s.size() == 0 || s[0] != '/')
|
||||
throw FormatError("bad path '%1%' in derivation", s);
|
||||
auto s = parseString(str);
|
||||
validatePath(s);
|
||||
return s;
|
||||
}
|
||||
|
||||
|
|
@ -140,7 +150,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
|
|||
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store,
|
||||
StorePath path, std::string_view hashAlgo, std::string_view hash)
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
|
||||
{
|
||||
if (hashAlgo != "") {
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
|
|
@ -148,40 +158,45 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
|||
method = FileIngestionMethod::Recursive;
|
||||
hashAlgo = hashAlgo.substr(2);
|
||||
}
|
||||
const HashType hashType = parseHashType(hashAlgo);
|
||||
|
||||
return hash != ""
|
||||
? DerivationOutput {
|
||||
.output = DerivationOutputCAFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
||||
},
|
||||
}
|
||||
}
|
||||
: (settings.requireExperimentalFeature("ca-derivations"),
|
||||
DerivationOutput {
|
||||
.output = DerivationOutputCAFloating {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
},
|
||||
});
|
||||
} else
|
||||
const auto hashType = parseHashType(hashAlgo);
|
||||
if (hash != "") {
|
||||
validatePath(pathS);
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputCAFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
settings.requireExperimentalFeature("ca-derivations");
|
||||
assert(pathS == "");
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputCAFloating {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
},
|
||||
};
|
||||
}
|
||||
} else {
|
||||
validatePath(pathS);
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = std::move(path),
|
||||
.path = store.parseStorePath(pathS),
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
|
||||
{
|
||||
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
|
||||
expect(str, ","); const auto pathS = parseString(str);
|
||||
expect(str, ","); const auto hashAlgo = parseString(str);
|
||||
expect(str, ","); const auto hash = parseString(str);
|
||||
expect(str, ")");
|
||||
|
||||
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -294,17 +309,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
for (auto & i : outputs) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, i.first);
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
|
||||
std::visit(overloaded {
|
||||
[&](DerivationOutputInputAddressed doi) {
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
[&](DerivationOutputCAFixed dof) {
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
|
||||
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
|
||||
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
|
|
@ -360,6 +377,16 @@ bool isDerivation(const string & fileName)
|
|||
}
|
||||
|
||||
|
||||
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
|
||||
std::string res { drvName };
|
||||
if (outputName != "out") {
|
||||
res += "-";
|
||||
res += outputName;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
DerivationType BasicDerivation::type() const
|
||||
{
|
||||
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
|
||||
|
|
@ -452,12 +479,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
|
||||
case DerivationType::CAFixed: {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & i : drv.outputsAndPaths(store)) {
|
||||
auto & dof = std::get<DerivationOutputCAFixed>(i.second.first.output);
|
||||
for (const auto & i : drv.outputs) {
|
||||
auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
|
||||
auto hash = hashString(htSHA256, "fixed:out:"
|
||||
+ dof.hash.printMethodAlgo() + ":"
|
||||
+ dof.hash.hash.to_string(Base16, false) + ":"
|
||||
+ store.printStorePath(i.second.second));
|
||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
return outputHashes;
|
||||
|
|
@ -508,21 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
|
|||
}
|
||||
|
||||
|
||||
StorePathSet BasicDerivation::outputPaths(const Store & store) const
|
||||
{
|
||||
StorePathSet paths;
|
||||
for (auto & i : outputsAndPaths(store))
|
||||
paths.insert(i.second.second);
|
||||
return paths;
|
||||
}
|
||||
|
||||
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
||||
{
|
||||
auto path = store.parseStorePath(readString(in));
|
||||
const auto pathS = readString(in);
|
||||
const auto hashAlgo = readString(in);
|
||||
const auto hash = readString(in);
|
||||
|
||||
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
}
|
||||
|
||||
StringSet BasicDerivation::outputNames() const
|
||||
|
|
@ -533,23 +552,12 @@ StringSet BasicDerivation::outputNames() const
|
|||
return names;
|
||||
}
|
||||
|
||||
DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
|
||||
DerivationOutputsAndPaths outsAndPaths;
|
||||
for (auto output : outputs)
|
||||
outsAndPaths.insert(std::make_pair(
|
||||
output.first,
|
||||
std::make_pair(output.second, output.second.path(store, name))
|
||||
)
|
||||
);
|
||||
return outsAndPaths;
|
||||
}
|
||||
|
||||
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
|
||||
DerivationOutputsAndOptPaths outsAndOptPaths;
|
||||
for (auto output : outputs)
|
||||
outsAndOptPaths.insert(std::make_pair(
|
||||
output.first,
|
||||
std::make_pair(output.second, output.second.pathOpt(store, output.first))
|
||||
std::make_pair(output.second, output.second.path(store, name, output.first))
|
||||
)
|
||||
);
|
||||
return outsAndOptPaths;
|
||||
|
|
@ -576,7 +584,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
|
|||
drv.outputs.emplace(std::move(name), std::move(output));
|
||||
}
|
||||
|
||||
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
|
||||
drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {});
|
||||
in >> drv.platform >> drv.builder;
|
||||
drv.args = readStrings<Strings>(in);
|
||||
|
||||
|
|
@ -594,24 +602,27 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
|
|||
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
|
||||
{
|
||||
out << drv.outputs.size();
|
||||
for (auto & i : drv.outputsAndPaths(store)) {
|
||||
out << i.first
|
||||
<< store.printStorePath(i.second.second);
|
||||
for (auto & i : drv.outputs) {
|
||||
out << i.first;
|
||||
std::visit(overloaded {
|
||||
[&](DerivationOutputInputAddressed doi) {
|
||||
out << "" << "";
|
||||
out << store.printStorePath(doi.path)
|
||||
<< ""
|
||||
<< "";
|
||||
},
|
||||
[&](DerivationOutputCAFixed dof) {
|
||||
out << dof.hash.printMethodAlgo()
|
||||
out << store.printStorePath(dof.path(store, drv.name, i.first))
|
||||
<< dof.hash.printMethodAlgo()
|
||||
<< dof.hash.hash.to_string(Base16, false);
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
||||
out << ""
|
||||
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
||||
<< "";
|
||||
},
|
||||
}, i.second.first.output);
|
||||
}, i.second.output);
|
||||
}
|
||||
writeStorePaths(store, out, drv.inputSrcs);
|
||||
worker_proto::write(store, out, drv.inputSrcs);
|
||||
out << drv.platform << drv.builder << drv.args;
|
||||
out << drv.env.size();
|
||||
for (auto & i : drv.env)
|
||||
|
|
@ -625,5 +636,65 @@ std::string hashPlaceholder(const std::string & outputName)
|
|||
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
|
||||
}
|
||||
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
|
||||
{
|
||||
auto drvNameWithExtension = drvPath.name();
|
||||
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
|
||||
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
|
||||
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
|
||||
}
|
||||
|
||||
|
||||
// N.B. Outputs are left unchanged
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
|
||||
|
||||
debug("Rewriting the derivation");
|
||||
|
||||
for (auto &rewrite: rewrites) {
|
||||
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
||||
}
|
||||
|
||||
drv.builder = rewriteStrings(drv.builder, rewrites);
|
||||
for (auto & arg: drv.args) {
|
||||
arg = rewriteStrings(arg, rewrites);
|
||||
}
|
||||
|
||||
StringPairs newEnv;
|
||||
for (auto & envVar: drv.env) {
|
||||
auto envName = rewriteStrings(envVar.first, rewrites);
|
||||
auto envValue = rewriteStrings(envVar.second, rewrites);
|
||||
newEnv.emplace(envName, envValue);
|
||||
}
|
||||
drv.env = newEnv;
|
||||
}
|
||||
|
||||
|
||||
Sync<DrvPathResolutions> drvPathResolutions;
|
||||
|
||||
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
|
||||
BasicDerivation resolved { *this };
|
||||
|
||||
// Input paths that we'll want to rewrite in the derivation
|
||||
StringMap inputRewrites;
|
||||
|
||||
for (auto & input : inputDrvs) {
|
||||
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
|
||||
StringSet newOutputNames;
|
||||
for (auto & outputName : input.second) {
|
||||
auto actualPathOpt = inputDrvOutputs.at(outputName);
|
||||
if (!actualPathOpt)
|
||||
return std::nullopt;
|
||||
auto actualPath = *actualPathOpt;
|
||||
inputRewrites.emplace(
|
||||
downstreamPlaceholder(store, input.first, outputName),
|
||||
store.printStorePath(actualPath));
|
||||
resolved.inputSrcs.insert(std::move(actualPath));
|
||||
}
|
||||
}
|
||||
|
||||
rewriteDerivation(store, resolved, inputRewrites);
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "types.hh"
|
||||
#include "hash.hh"
|
||||
#include "content-address.hh"
|
||||
#include "sync.hh"
|
||||
|
||||
#include <map>
|
||||
#include <variant>
|
||||
|
|
@ -27,6 +28,7 @@ struct DerivationOutputInputAddressed
|
|||
struct DerivationOutputCAFixed
|
||||
{
|
||||
FixedOutputHash hash; /* hash used for expected hash computation */
|
||||
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
|
||||
};
|
||||
|
||||
/* Floating-output derivations, whose output paths are content addressed, but
|
||||
|
|
@ -49,14 +51,8 @@ struct DerivationOutput
|
|||
std::optional<HashType> hashAlgoOpt(const Store & store) const;
|
||||
/* Note, when you use this function you should make sure that you're passing
|
||||
the right derivation name. When in doubt, you should use the safer
|
||||
interface provided by BasicDerivation::outputsAndPaths */
|
||||
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
|
||||
/* DEPRECATED: Remove after CA drvs are fully implemented */
|
||||
StorePath path(const Store & store, std::string_view drvName) const {
|
||||
auto p = pathOpt(store, drvName);
|
||||
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
|
||||
return *p;
|
||||
}
|
||||
interface provided by BasicDerivation::outputsAndOptPaths */
|
||||
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
|
||||
};
|
||||
|
||||
typedef std::map<string, DerivationOutput> DerivationOutputs;
|
||||
|
|
@ -65,8 +61,6 @@ typedef std::map<string, DerivationOutput> DerivationOutputs;
|
|||
also contains, for each output, the (optional) store path in which it would
|
||||
be written. To calculate values of these types, see the corresponding
|
||||
functions in BasicDerivation */
|
||||
typedef std::map<string, std::pair<DerivationOutput, StorePath>>
|
||||
DerivationOutputsAndPaths;
|
||||
typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
|
||||
DerivationOutputsAndOptPaths;
|
||||
|
||||
|
|
@ -105,7 +99,7 @@ struct BasicDerivation
|
|||
StringPairs env;
|
||||
std::string name;
|
||||
|
||||
BasicDerivation() { }
|
||||
BasicDerivation() = default;
|
||||
virtual ~BasicDerivation() { };
|
||||
|
||||
bool isBuiltin() const;
|
||||
|
|
@ -113,17 +107,12 @@ struct BasicDerivation
|
|||
/* Return true iff this is a fixed-output derivation. */
|
||||
DerivationType type() const;
|
||||
|
||||
/* Return the output paths of a derivation. */
|
||||
StorePathSet outputPaths(const Store & store) const;
|
||||
|
||||
/* Return the output names of a derivation. */
|
||||
StringSet outputNames() const;
|
||||
|
||||
/* Calculates the maps that contains all the DerivationOutputs, but
|
||||
augmented with knowledge of the Store paths they would be written into.
|
||||
The first one of these functions will be removed when the CA work is
|
||||
completed */
|
||||
DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
|
||||
augmented with knowledge of the Store paths they would be written
|
||||
into. */
|
||||
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
|
||||
|
||||
static std::string_view nameFromPath(const StorePath & storePath);
|
||||
|
|
@ -137,7 +126,17 @@ struct Derivation : BasicDerivation
|
|||
std::string unparse(const Store & store, bool maskOutputs,
|
||||
std::map<std::string, StringSet> * actualInputs = nullptr) const;
|
||||
|
||||
Derivation() { }
|
||||
/* Return the underlying basic derivation but with these changes:
|
||||
|
||||
1. Input drvs are emptied, but the outputs of them that were used are
|
||||
added directly to input sources.
|
||||
|
||||
2. Input placeholders are replaced with realized input store paths. */
|
||||
std::optional<BasicDerivation> tryResolve(Store & store);
|
||||
|
||||
Derivation() = default;
|
||||
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
|
||||
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -147,7 +146,9 @@ enum RepairFlag : bool { NoRepair = false, Repair = true };
|
|||
|
||||
/* Write a derivation to the Nix store, and return its path. */
|
||||
StorePath writeDerivation(Store & store,
|
||||
const Derivation & drv, RepairFlag repair = NoRepair);
|
||||
const Derivation & drv,
|
||||
RepairFlag repair = NoRepair,
|
||||
bool readOnly = false);
|
||||
|
||||
/* Read a derivation from a file. */
|
||||
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
|
||||
|
|
@ -155,6 +156,13 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
|
|||
// FIXME: remove
|
||||
bool isDerivation(const string & fileName);
|
||||
|
||||
/* Calculate the name that will be used for the store path for this
|
||||
output.
|
||||
|
||||
This is usually <drv-name>-<output-name>, but is just <drv-name> when
|
||||
the output name is "out". */
|
||||
std::string outputPathName(std::string_view drvName, std::string_view outputName);
|
||||
|
||||
// known CA drv's output hashes, current just for fixed-output derivations
|
||||
// whose output hashes are always known since they are fixed up-front.
|
||||
typedef std::map<std::string, Hash> CaOutputHashes;
|
||||
|
|
@ -194,6 +202,16 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
|
|||
|
||||
extern DrvHashes drvHashes; // FIXME: global, not thread-safe
|
||||
|
||||
/* Memoisation of `readDerivation(..).resove()`. */
|
||||
typedef std::map<
|
||||
StorePath,
|
||||
std::optional<StorePath>
|
||||
> DrvPathResolutions;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
// FIXME: arguably overlaps with hashDerivationModulo memo table.
|
||||
extern Sync<DrvPathResolutions> drvPathResolutions;
|
||||
|
||||
bool wantOutput(const string & output, const std::set<string> & wanted);
|
||||
|
||||
struct Source;
|
||||
|
|
@ -202,6 +220,21 @@ struct Sink;
|
|||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
|
||||
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
|
||||
|
||||
/* This creates an opaque and almost certainly unique string
|
||||
deterministically from the output name.
|
||||
|
||||
It is used as a placeholder to allow derivations to refer to their
|
||||
own outputs without needing to use the hash of a derivation in
|
||||
itself, making the hash near-impossible to calculate. */
|
||||
std::string hashPlaceholder(const std::string & outputName);
|
||||
|
||||
/* This creates an opaque and almost certainly unique string
|
||||
deterministically from a derivation path and output name.
|
||||
|
||||
It is used as a placeholder to allow derivations to refer to
|
||||
content-addressed paths whose content --- and thus the path
|
||||
themselves --- isn't yet known. This occurs when a derivation has a
|
||||
dependency which is a CA derivation. */
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,28 @@
|
|||
#include "store-api.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "dummy://";
|
||||
struct DummyStoreConfig : virtual StoreConfig {
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
struct DummyStore : public Store
|
||||
const std::string name() override { return "Dummy Store"; }
|
||||
};
|
||||
|
||||
struct DummyStore : public Store, public virtual DummyStoreConfig
|
||||
{
|
||||
DummyStore(const std::string scheme, const std::string uri, const Params & params)
|
||||
: DummyStore(params)
|
||||
{ }
|
||||
|
||||
DummyStore(const Params & params)
|
||||
: Store(params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
{ }
|
||||
|
||||
string getUri() override
|
||||
{
|
||||
return uriScheme;
|
||||
return *uriSchemes().begin();
|
||||
}
|
||||
|
||||
void queryPathInfoUncached(const StorePath & path,
|
||||
|
|
@ -21,6 +31,10 @@ struct DummyStore : public Store
|
|||
callback(nullptr);
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() {
|
||||
return {"dummy"};
|
||||
}
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
||||
{ unsupported("queryPathFromHashPart"); }
|
||||
|
||||
|
|
@ -48,12 +62,6 @@ struct DummyStore : public Store
|
|||
{ unsupported("buildDerivation"); }
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (uri != uriScheme) return nullptr;
|
||||
return std::make_shared<DummyStore>(params);
|
||||
});
|
||||
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
|
|||
teeSink
|
||||
<< exportMagic
|
||||
<< printStorePath(path);
|
||||
writeStorePaths(*this, teeSink, info->references);
|
||||
worker_proto::write(*this, teeSink, info->references);
|
||||
teeSink
|
||||
<< (info->deriver ? printStorePath(*info->deriver) : "")
|
||||
<< 0;
|
||||
|
|
@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
|
|||
|
||||
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
|
||||
|
||||
auto references = readStorePaths<StorePathSet>(*this, source);
|
||||
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
|
||||
auto deriver = readString(source);
|
||||
auto narHash = hashString(htSHA256, *saved.s);
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "s3.hh"
|
||||
#include "compression.hh"
|
||||
#include "finally.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#ifdef ENABLE_S3
|
||||
#include <aws/core/client/ClientConfiguration.h>
|
||||
|
|
@ -30,7 +31,7 @@ namespace nix {
|
|||
|
||||
FileTransferSettings fileTransferSettings;
|
||||
|
||||
static GlobalConfig::Register r1(&fileTransferSettings);
|
||||
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
|
||||
|
||||
std::string resolveUri(const std::string & uri)
|
||||
{
|
||||
|
|
@ -112,6 +113,9 @@ struct curlFileTransfer : public FileTransfer
|
|||
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
||||
if (!request.mimeType.empty())
|
||||
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
||||
for (auto it = request.headers.begin(); it != request.headers.end(); ++it){
|
||||
requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
~TransferItem()
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ extern FileTransferSettings fileTransferSettings;
|
|||
struct FileTransferRequest
|
||||
{
|
||||
std::string uri;
|
||||
Headers headers;
|
||||
std::string expectedETag;
|
||||
bool verifyTLS = true;
|
||||
bool head = false;
|
||||
|
|
|
|||
|
|
@ -574,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
|
|||
/* If keep-derivations is set and this is a derivation, then
|
||||
don't delete the derivation if any of the outputs are alive. */
|
||||
if (state.gcKeepDerivations && path.isDerivation()) {
|
||||
for (auto & i : queryDerivationOutputs(path))
|
||||
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
|
||||
incoming.insert(i);
|
||||
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
||||
if (maybeOutPath &&
|
||||
isValidPath(*maybeOutPath) &&
|
||||
queryPathInfo(*maybeOutPath)->deriver == path
|
||||
)
|
||||
incoming.insert(*maybeOutPath);
|
||||
}
|
||||
|
||||
/* If keep-outputs is set, then don't delete this path if there
|
||||
|
|
@ -660,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
|||
if (name == "." || name == "..") continue;
|
||||
Path path = linksDir + "/" + name;
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st) == -1)
|
||||
throw SysError("statting '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (st.st_nlink != 1) {
|
||||
actualSize += st.st_size;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
#include "args.hh"
|
||||
#include "abstract-setting-to-json.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
|
|
@ -25,7 +26,7 @@ namespace nix {
|
|||
|
||||
Settings settings;
|
||||
|
||||
static GlobalConfig::Register r1(&settings);
|
||||
static GlobalConfig::Register rSettings(&settings);
|
||||
|
||||
Settings::Settings()
|
||||
: nixPrefix(NIX_PREFIX)
|
||||
|
|
@ -42,6 +43,7 @@ Settings::Settings()
|
|||
{
|
||||
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
|
||||
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
|
||||
allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
|
||||
|
||||
caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
|
||||
if (caFile == "") {
|
||||
|
|
@ -147,6 +149,12 @@ bool Settings::isWSL1()
|
|||
|
||||
const string nixVersion = PACKAGE_VERSION;
|
||||
|
||||
NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
|
||||
{SandboxMode::smEnabled, true},
|
||||
{SandboxMode::smRelaxed, "relaxed"},
|
||||
{SandboxMode::smDisabled, false},
|
||||
});
|
||||
|
||||
template<> void BaseSetting<SandboxMode>::set(const std::string & str)
|
||||
{
|
||||
if (str == "true") value = smEnabled;
|
||||
|
|
@ -163,11 +171,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
|
|||
else abort();
|
||||
}
|
||||
|
||||
template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
|
||||
{
|
||||
return AbstractSetting::toJSON();
|
||||
}
|
||||
|
||||
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
|
||||
{
|
||||
args.addFlag({
|
||||
|
|
|
|||
|
|
@ -859,8 +859,54 @@ public:
|
|||
are loaded as plugins (non-recursively).
|
||||
)"};
|
||||
|
||||
Setting<std::string> githubAccessToken{this, "", "github-access-token",
|
||||
"GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
|
||||
Setting<StringMap> accessTokens{this, {}, "access-tokens",
|
||||
R"(
|
||||
Access tokens used to access protected GitHub, GitLab, or
|
||||
other locations requiring token-based authentication.
|
||||
|
||||
Access tokens are specified as a string made up of
|
||||
space-separated `host=token` values. The specific token
|
||||
used is selected by matching the `host` portion against the
|
||||
"host" specification of the input. The actual use of the
|
||||
`token` value is determined by the type of resource being
|
||||
accessed:
|
||||
|
||||
* Github: the token value is the OAUTH-TOKEN string obtained
|
||||
as the Personal Access Token from the Github server (see
|
||||
https://docs.github.com/en/developers/apps/authorizing-oath-apps).
|
||||
|
||||
* Gitlab: the token value is either the OAuth2 token or the
|
||||
Personal Access Token (these are different types tokens
|
||||
for gitlab, see
|
||||
https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
|
||||
The `token` value should be `type:tokenstring` where
|
||||
`type` is either `OAuth2` or `PAT` to indicate which type
|
||||
of token is being specified.
|
||||
|
||||
Example `~/.config/nix/nix.conf`:
|
||||
|
||||
```
|
||||
access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
|
||||
```
|
||||
|
||||
Example `~/code/flake.nix`:
|
||||
|
||||
```nix
|
||||
input.foo = {
|
||||
type = "gitlab";
|
||||
host = "gitlab.mycompany.com";
|
||||
owner = "mycompany";
|
||||
repo = "pro";
|
||||
};
|
||||
```
|
||||
|
||||
This example specifies three tokens, one each for accessing
|
||||
github.com, gitlab.mycompany.com, and sourceforge.net.
|
||||
|
||||
The `input.foo` uses the "gitlab" fetcher, which might
|
||||
requires specifying the token type along with the token
|
||||
value.
|
||||
)"};
|
||||
|
||||
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
|
||||
"Experimental Nix features to enable."};
|
||||
|
|
@ -880,6 +926,19 @@ public:
|
|||
|
||||
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
|
||||
"Path or URI of the global flake registry."};
|
||||
|
||||
Setting<bool> allowSymlinkedStore{
|
||||
this, false, "allow-symlinked-store",
|
||||
R"(
|
||||
If set to `true`, Nix will stop complaining if the store directory
|
||||
(typically /nix/store) contains symlink components.
|
||||
|
||||
This risks making some builds "impure" because builders sometimes
|
||||
"canonicalise" paths by resolving all symlink components. Problems
|
||||
occur if those builds are then deployed to machines where /nix/store
|
||||
resolves to a different location from that of the build machine. You
|
||||
can enable this setting if you are sure you're not going to do that.
|
||||
)"};
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,20 @@
|
|||
#include "filetransfer.hh"
|
||||
#include "globals.hh"
|
||||
#include "nar-info-disk-cache.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
MakeError(UploadToHTTP, Error);
|
||||
|
||||
class HttpBinaryCacheStore : public BinaryCacheStore
|
||||
struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
|
||||
const std::string name() override { return "Http Binary Cache Store"; }
|
||||
};
|
||||
|
||||
class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -24,9 +32,12 @@ private:
|
|||
public:
|
||||
|
||||
HttpBinaryCacheStore(
|
||||
const Params & params, const Path & _cacheUri)
|
||||
: BinaryCacheStore(params)
|
||||
, cacheUri(_cacheUri)
|
||||
const std::string & scheme,
|
||||
const Path & _cacheUri,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStore(params)
|
||||
, cacheUri(scheme + "://" + _cacheUri)
|
||||
{
|
||||
if (cacheUri.back() == '/')
|
||||
cacheUri.pop_back();
|
||||
|
|
@ -55,6 +66,14 @@ public:
|
|||
}
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
|
||||
auto ret = std::set<std::string>({"http", "https"});
|
||||
if (forceHttp) ret.insert("file");
|
||||
return ret;
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
void maybeDisable()
|
||||
|
|
@ -162,18 +181,6 @@ protected:
|
|||
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
|
||||
if (std::string(uri, 0, 7) != "http://" &&
|
||||
std::string(uri, 0, 8) != "https://" &&
|
||||
(!forceHttp || std::string(uri, 0, 7) != "file://"))
|
||||
return 0;
|
||||
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,21 +6,28 @@
|
|||
#include "worker-protocol.hh"
|
||||
#include "ssh.hh"
|
||||
#include "derivations.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "ssh://";
|
||||
|
||||
struct LegacySSHStore : public Store
|
||||
struct LegacySSHStoreConfig : virtual StoreConfig
|
||||
{
|
||||
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
|
||||
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
|
||||
using StoreConfig::StoreConfig;
|
||||
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
|
||||
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
const std::string name() override { return "Legacy SSH Store"; }
|
||||
};
|
||||
|
||||
struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
|
||||
{
|
||||
// Hack for getting remote build log output.
|
||||
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
|
||||
// the documentation
|
||||
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
|
||||
struct Connection
|
||||
{
|
||||
|
|
@ -37,8 +44,11 @@ struct LegacySSHStore : public Store
|
|||
|
||||
SSHMaster master;
|
||||
|
||||
LegacySSHStore(const string & host, const Params & params)
|
||||
: Store(params)
|
||||
static std::set<std::string> uriSchemes() { return {"ssh"}; }
|
||||
|
||||
LegacySSHStore(const string & scheme, const string & host, const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, host(host)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
|
|
@ -84,7 +94,7 @@ struct LegacySSHStore : public Store
|
|||
|
||||
string getUri() override
|
||||
{
|
||||
return uriScheme + host;
|
||||
return *uriSchemes().begin() + "://" + host;
|
||||
}
|
||||
|
||||
void queryPathInfoUncached(const StorePath & path,
|
||||
|
|
@ -112,7 +122,7 @@ struct LegacySSHStore : public Store
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info->deriver = parseStorePath(deriver);
|
||||
info->references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
readLongLong(conn->from); // download size
|
||||
info->narSize = readLongLong(conn->from);
|
||||
|
||||
|
|
@ -146,7 +156,7 @@ struct LegacySSHStore : public Store
|
|||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
writeStorePaths(*this, conn->to, info.references);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
conn->to
|
||||
<< info.registrationTime
|
||||
<< info.narSize
|
||||
|
|
@ -175,7 +185,7 @@ struct LegacySSHStore : public Store
|
|||
conn->to
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
writeStorePaths(*this, conn->to, info.references);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
conn->to
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0
|
||||
|
|
@ -291,10 +301,10 @@ public:
|
|||
conn->to
|
||||
<< cmdQueryClosure
|
||||
<< includeOutputs;
|
||||
writeStorePaths(*this, conn->to, paths);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
conn->to.flush();
|
||||
|
||||
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
|
||||
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
|
||||
out.insert(i);
|
||||
}
|
||||
|
||||
|
|
@ -307,10 +317,10 @@ public:
|
|||
<< cmdQueryValidPaths
|
||||
<< false // lock
|
||||
<< maybeSubstitute;
|
||||
writeStorePaths(*this, conn->to, paths);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
conn->to.flush();
|
||||
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
|
||||
void connect() override
|
||||
|
|
@ -325,12 +335,6 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
|
||||
});
|
||||
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,14 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
class LocalBinaryCacheStore : public BinaryCacheStore
|
||||
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
|
||||
const std::string name() override { return "Local Binary Cache Store"; }
|
||||
};
|
||||
|
||||
class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -13,8 +20,11 @@ private:
|
|||
public:
|
||||
|
||||
LocalBinaryCacheStore(
|
||||
const Params & params, const Path & binaryCacheDir)
|
||||
: BinaryCacheStore(params)
|
||||
const std::string scheme,
|
||||
const Path & binaryCacheDir,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStore(params)
|
||||
, binaryCacheDir(binaryCacheDir)
|
||||
{
|
||||
}
|
||||
|
|
@ -26,6 +36,8 @@ public:
|
|||
return "file://" + binaryCacheDir;
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes();
|
||||
|
||||
protected:
|
||||
|
||||
bool fileExists(const std::string & path) override;
|
||||
|
|
@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
|
|||
return pathExists(binaryCacheDir + "/" + path);
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
std::set<std::string> LocalBinaryCacheStore::uriSchemes()
|
||||
{
|
||||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
|
||||
std::string(uri, 0, 7) != "file://")
|
||||
return 0;
|
||||
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
|
||||
return {};
|
||||
else
|
||||
return {"file"};
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "derivations.hh"
|
||||
#include "nar-info.hh"
|
||||
#include "references.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
|
|
@ -42,7 +43,8 @@ namespace nix {
|
|||
|
||||
|
||||
LocalStore::LocalStore(const Params & params)
|
||||
: Store(params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
|
||||
"physical path to the Nix store"}
|
||||
|
|
@ -108,12 +110,11 @@ LocalStore::LocalStore(const Params & params)
|
|||
}
|
||||
|
||||
/* Ensure that the store and its parents are not symlinks. */
|
||||
if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
|
||||
if (!settings.allowSymlinkedStore) {
|
||||
Path path = realStoreDir;
|
||||
struct stat st;
|
||||
while (path != "/") {
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting status of '%1%'", path);
|
||||
st = lstat(path);
|
||||
if (S_ISLNK(st.st_mode))
|
||||
throw Error(
|
||||
"the path '%1%' is a symlink; "
|
||||
|
|
@ -417,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
|
|||
|
||||
void canonicaliseTimestampAndPermissions(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
canonicaliseTimestampAndPermissions(path, st);
|
||||
canonicaliseTimestampAndPermissions(path, lstat(path));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -438,9 +436,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
|
|||
}
|
||||
#endif
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
/* Really make sure that the path is of a supported type. */
|
||||
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
|
||||
|
|
@ -476,8 +472,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
|
|||
ensure that we don't fail on hard links within the same build
|
||||
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */
|
||||
if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
|
||||
assert(!S_ISDIR(st.st_mode));
|
||||
if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
|
||||
if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
|
||||
throw BuildError("invalid ownership on file '%1%'", path);
|
||||
mode_t mode = st.st_mode & ~S_IFMT;
|
||||
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
|
||||
|
|
@ -520,9 +515,7 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
|
|||
|
||||
/* On platforms that don't have lchown(), the top-level path can't
|
||||
be a symlink, since we can't change its ownership. */
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (st.st_uid != geteuid()) {
|
||||
assert(S_ISLNK(st.st_mode));
|
||||
|
|
@ -578,13 +571,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
envHasRightPath(path, i.first);
|
||||
},
|
||||
[&](DerivationOutputCAFloating _) {
|
||||
throw UnimplementedError("floating CA output derivations are not yet implemented");
|
||||
/* Nothing to check */
|
||||
},
|
||||
}, i.second.output);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
|
||||
{
|
||||
auto state(_state.lock());
|
||||
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
|
||||
}
|
||||
|
||||
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(deriver)
|
||||
(outputName)
|
||||
(printStorePath(output))
|
||||
.exec();
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
uint64_t LocalStore::addValidPath(State & state,
|
||||
const ValidPathInfo & info, bool checkOutputs)
|
||||
{
|
||||
|
|
@ -618,12 +630,11 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
registration above is undone. */
|
||||
if (checkOutputs) checkDerivationOutputs(info.path, drv);
|
||||
|
||||
for (auto & i : drv.outputsAndPaths(*this)) {
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(id)
|
||||
(i.first)
|
||||
(printStorePath(i.second.second))
|
||||
.exec();
|
||||
for (auto & i : drv.outputsAndOptPaths(*this)) {
|
||||
/* Floating CA derivations have indeterminate output paths until
|
||||
they are built, so don't register anything in that case */
|
||||
if (i.second.second)
|
||||
linkDeriverToPath(state, id, i.first, *i.second.second);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -710,7 +721,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
|
|||
{
|
||||
auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
|
||||
if (!use.next())
|
||||
throw Error("path '%s' is not valid", printStorePath(path));
|
||||
throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
||||
return use.getInt(0);
|
||||
}
|
||||
|
||||
|
|
@ -785,18 +796,58 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
|
|||
}
|
||||
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path)
|
||||
std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
|
||||
{
|
||||
auto path = path_;
|
||||
std::map<std::string, std::optional<StorePath>> outputs;
|
||||
BasicDerivation drv = readDerivation(path);
|
||||
Derivation drv = readDerivation(path);
|
||||
for (auto & [outName, _] : drv.outputs) {
|
||||
outputs.insert_or_assign(outName, std::nullopt);
|
||||
}
|
||||
bool haveCached = false;
|
||||
{
|
||||
auto resolutions = drvPathResolutions.lock();
|
||||
auto resolvedPathOptIter = resolutions->find(path);
|
||||
if (resolvedPathOptIter != resolutions->end()) {
|
||||
auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
|
||||
if (resolvedPathOpt)
|
||||
path = *resolvedPathOpt;
|
||||
haveCached = true;
|
||||
}
|
||||
}
|
||||
/* can't just use else-if instead of `!haveCached` because we need to unlock
|
||||
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
|
||||
if (!haveCached && drv.type() == DerivationType::CAFloating) {
|
||||
/* Try resolve drv and use that path instead. */
|
||||
auto attempt = drv.tryResolve(*this);
|
||||
if (!attempt)
|
||||
/* If we cannot resolve the derivation, we cannot have any path
|
||||
assigned so we return the map of all std::nullopts. */
|
||||
return outputs;
|
||||
/* Just compute store path */
|
||||
auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
|
||||
/* Store in memo table. */
|
||||
/* FIXME: memo logic should not be local-store specific, should have
|
||||
wrapper-method instead. */
|
||||
drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
|
||||
path = std::move(pathResolved);
|
||||
}
|
||||
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
|
||||
(queryValidPathId(*state, path)));
|
||||
uint64_t drvId;
|
||||
try {
|
||||
drvId = queryValidPathId(*state, path);
|
||||
} catch (InvalidPath &) {
|
||||
/* FIXME? if the derivation doesn't exist, we cannot have a mapping
|
||||
for it. */
|
||||
return outputs;
|
||||
}
|
||||
|
||||
auto useQueryDerivationOutputs {
|
||||
state->stmtQueryDerivationOutputs.use()
|
||||
(drvId)
|
||||
};
|
||||
|
||||
while (useQueryDerivationOutputs.next())
|
||||
outputs.insert_or_assign(
|
||||
|
|
@ -1435,7 +1486,7 @@ static void makeMutable(const Path & path)
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
struct stat st = lstat(path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,19 @@ struct OptimiseStats
|
|||
uint64_t blocksFreed = 0;
|
||||
};
|
||||
|
||||
struct LocalStoreConfig : virtual LocalFSStoreConfig
|
||||
{
|
||||
using LocalFSStoreConfig::LocalFSStoreConfig;
|
||||
|
||||
class LocalStore : public LocalFSStore
|
||||
Setting<bool> requireSigs{(StoreConfig*) this,
|
||||
settings.requireSigs,
|
||||
"require-sigs", "whether store paths should have a trusted signature on import"};
|
||||
|
||||
const std::string name() override { return "Local Store"; }
|
||||
};
|
||||
|
||||
|
||||
class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -95,10 +106,6 @@ public:
|
|||
|
||||
private:
|
||||
|
||||
Setting<bool> requireSigs{(Store*) this,
|
||||
settings.requireSigs,
|
||||
"require-sigs", "whether store paths should have a trusted signature on import"};
|
||||
|
||||
const PublicKeys & getPublicKeys();
|
||||
|
||||
public:
|
||||
|
|
@ -279,6 +286,11 @@ private:
|
|||
specified by the ‘secret-key-files’ option. */
|
||||
void signPathInfo(ValidPathInfo & info);
|
||||
|
||||
/* Register the store path 'output' as the output named 'outputName' of
|
||||
derivation 'deriver'. */
|
||||
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
|
||||
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
|
||||
|
||||
Path getRealStoreDir() override { return realStoreDir; }
|
||||
|
||||
void createUser(const std::string & userName, uid_t userId) override;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
#include "store-api.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "topo-sort.hh"
|
||||
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
|||
return;
|
||||
}
|
||||
|
||||
PathSet invalid;
|
||||
/* true for regular derivations, and CA derivations for which we
|
||||
have a trust mapping for all wanted outputs. */
|
||||
auto knownOutputPaths = true;
|
||||
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
|
||||
if (!pathOpt) {
|
||||
knownOutputPaths = false;
|
||||
break;
|
||||
}
|
||||
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
|
||||
invalid.insert(printStorePath(*pathOpt));
|
||||
}
|
||||
if (knownOutputPaths && invalid.empty()) return;
|
||||
|
||||
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
|
||||
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
|
||||
|
||||
PathSet invalid;
|
||||
for (auto & j : drv->outputsAndPaths(*this))
|
||||
if (wantOutput(j.first, path.outputs)
|
||||
&& !isValidPath(j.second.second))
|
||||
invalid.insert(printStorePath(j.second.second));
|
||||
if (invalid.empty()) return;
|
||||
|
||||
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
|
||||
for (auto & output : invalid)
|
||||
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
|
||||
|
|
|
|||
|
|
@ -1,10 +1,18 @@
|
|||
#include "names.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
struct Regex
|
||||
{
|
||||
std::regex regex;
|
||||
};
|
||||
|
||||
|
||||
DrvName::DrvName()
|
||||
{
|
||||
name = "";
|
||||
|
|
@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
|
|||
}
|
||||
|
||||
|
||||
DrvName::~DrvName()
|
||||
{ }
|
||||
|
||||
|
||||
bool DrvName::matches(DrvName & n)
|
||||
{
|
||||
if (name != "*") {
|
||||
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
|
||||
if (!std::regex_match(n.name, *regex)) return false;
|
||||
if (!regex) {
|
||||
regex = std::make_unique<Regex>();
|
||||
regex->regex = std::regex(name, std::regex::extended);
|
||||
}
|
||||
if (!std::regex_match(n.name, regex->regex)) return false;
|
||||
}
|
||||
if (version != "" && version != n.version) return false;
|
||||
return true;
|
||||
|
|
@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
|
|||
{
|
||||
DrvNames result;
|
||||
for (auto & i : opArgs)
|
||||
result.push_back(DrvName(i));
|
||||
result.emplace_back(i);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@
|
|||
#include <memory>
|
||||
|
||||
#include "types.hh"
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct Regex;
|
||||
|
||||
struct DrvName
|
||||
{
|
||||
string fullName;
|
||||
|
|
@ -16,10 +17,12 @@ struct DrvName
|
|||
|
||||
DrvName();
|
||||
DrvName(std::string_view s);
|
||||
~DrvName();
|
||||
|
||||
bool matches(DrvName & n);
|
||||
|
||||
private:
|
||||
std::unique_ptr<std::regex> regex;
|
||||
std::unique_ptr<Regex> regex;
|
||||
};
|
||||
|
||||
typedef list<DrvName> DrvNames;
|
||||
|
|
|
|||
|
|
@ -17,9 +17,7 @@ namespace nix {
|
|||
|
||||
static void makeWritable(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||
throw SysError("changing writability of '%1%'", path);
|
||||
}
|
||||
|
|
@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
#if __APPLE__
|
||||
/* HFS/macOS has some undocumented security feature disabling hardlinking for
|
||||
|
|
@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
|||
|
||||
/* Yes! We've seen a file with the same contents. Replace the
|
||||
current file with a hard link to that file. */
|
||||
struct stat stLink;
|
||||
if (lstat(linkPath.c_str(), &stLink))
|
||||
throw SysError("getting attributes of path '%1%'", linkPath);
|
||||
auto stLink = lstat(linkPath);
|
||||
|
||||
if (st.st_ino == stLink.st_ino) {
|
||||
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
|
||||
|
|
@ -282,21 +276,15 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
|||
}
|
||||
}
|
||||
|
||||
static string showBytes(uint64_t bytes)
|
||||
{
|
||||
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
||||
}
|
||||
|
||||
void LocalStore::optimiseStore()
|
||||
{
|
||||
OptimiseStats stats;
|
||||
|
||||
optimiseStore(stats);
|
||||
|
||||
printInfo(
|
||||
format("%1% freed by hard-linking %2% files")
|
||||
% showBytes(stats.bytesFreed)
|
||||
% stats.filesLinked);
|
||||
printInfo("%s freed by hard-linking %d files",
|
||||
showBytes(stats.bytesFreed),
|
||||
stats.filesLinked);
|
||||
}
|
||||
|
||||
void LocalStore::optimisePath(const Path & path)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#pragma once
|
||||
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
|
|
|||
|
|
@ -39,13 +39,10 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
|
|||
for (auto & i : readDirectory(profileDir)) {
|
||||
if (auto n = parseName(profileName, i.name)) {
|
||||
auto path = profileDir + "/" + i.name;
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st) != 0)
|
||||
throw SysError("statting '%1%'", path);
|
||||
gens.push_back({
|
||||
.number = *n,
|
||||
.path = path,
|
||||
.creationTime = st.st_mtime
|
||||
.creationTime = lstat(path).st_mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
|
|||
std::pair<PathSet, HashResult> scanForReferences(const string & path,
|
||||
const PathSet & refs)
|
||||
{
|
||||
RefScanSink refsSink;
|
||||
HashSink hashSink { htSHA256 };
|
||||
TeeSink sink { refsSink, hashSink };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<PathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
PathSet scanForReferences(Sink & toTee,
|
||||
const string & path, const PathSet & refs)
|
||||
{
|
||||
RefScanSink refsSink;
|
||||
TeeSink sink { refsSink, toTee };
|
||||
std::map<string, Path> backMap;
|
||||
|
||||
/* For efficiency (and a higher hit rate), just search for the
|
||||
|
|
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
|
|||
found.insert(j->second);
|
||||
}
|
||||
|
||||
auto hash = hashSink.finish();
|
||||
|
||||
return std::pair<PathSet, HashResult>(found, hash);
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ namespace nix {
|
|||
|
||||
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
|
||||
|
||||
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
|
||||
|
||||
struct RewritingSink : Sink
|
||||
{
|
||||
std::string from, to, prev;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "serialise.hh"
|
||||
#include "util.hh"
|
||||
#include "remote-fs-accessor.hh"
|
||||
#include "remote-store.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -9,6 +10,7 @@
|
|||
#include "pool.hh"
|
||||
#include "finally.hh"
|
||||
#include "logging.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
|
|
@ -22,44 +24,19 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
|
||||
template<> StorePathSet readStorePaths(const Store & store, Source & from)
|
||||
{
|
||||
StorePathSet paths;
|
||||
for (auto & i : readStrings<Strings>(from))
|
||||
paths.insert(store.parseStorePath(i));
|
||||
return paths;
|
||||
}
|
||||
|
||||
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
|
||||
{
|
||||
out << paths.size();
|
||||
for (auto & i : paths)
|
||||
out << store.printStorePath(i);
|
||||
}
|
||||
|
||||
|
||||
StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
|
||||
{
|
||||
StorePathCAMap paths;
|
||||
auto count = readNum<size_t>(from);
|
||||
while (count--)
|
||||
paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
|
||||
return paths;
|
||||
}
|
||||
|
||||
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
|
||||
{
|
||||
out << paths.size();
|
||||
for (auto & i : paths) {
|
||||
out << store.printStorePath(i.first);
|
||||
out << renderContentAddress(i.second);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
namespace worker_proto {
|
||||
|
||||
std::string read(const Store & store, Source & from, Phantom<std::string> _)
|
||||
{
|
||||
return readString(from);
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::string & str)
|
||||
{
|
||||
out << str;
|
||||
}
|
||||
|
||||
|
||||
StorePath read(const Store & store, Source & from, Phantom<StorePath> _)
|
||||
{
|
||||
return store.parseStorePath(readString(from));
|
||||
|
|
@ -71,28 +48,58 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
|
|||
}
|
||||
|
||||
|
||||
template<>
|
||||
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
|
||||
{
|
||||
return parseContentAddress(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const ContentAddress & ca)
|
||||
{
|
||||
out << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
|
||||
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
|
||||
{
|
||||
auto s = readString(from);
|
||||
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
||||
}
|
||||
|
||||
template<>
|
||||
void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt)
|
||||
{
|
||||
out << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
||||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
|
||||
{
|
||||
return parseContentAddressOpt(readString(from));
|
||||
}
|
||||
|
||||
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
|
||||
{
|
||||
out << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
/* TODO: Separate these store impls into different files, give them better names */
|
||||
RemoteStore::RemoteStore(const Params & params)
|
||||
: Store(params)
|
||||
, RemoteStoreConfig(params)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
[this]() { return openConnectionWrapper(); },
|
||||
[this]() {
|
||||
auto conn = openConnectionWrapper();
|
||||
try {
|
||||
initConnection(*conn);
|
||||
} catch (...) {
|
||||
failed = true;
|
||||
throw;
|
||||
}
|
||||
return conn;
|
||||
},
|
||||
[this](const ref<Connection> & r) {
|
||||
return
|
||||
r->to.good()
|
||||
|
|
@ -119,19 +126,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
|||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(const Params & params)
|
||||
: Store(params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
|
||||
: Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
, path(socket_path)
|
||||
UDSRemoteStore::UDSRemoteStore(
|
||||
const std::string scheme,
|
||||
std::string socket_path,
|
||||
const Params & params)
|
||||
: UDSRemoteStore(params)
|
||||
{
|
||||
path.emplace(socket_path);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -175,8 +184,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
|||
|
||||
conn->startTime = std::chrono::steady_clock::now();
|
||||
|
||||
initConnection(*conn);
|
||||
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
|
@ -292,6 +299,8 @@ struct ConnectionHandle
|
|||
std::rethrow_exception(ex);
|
||||
}
|
||||
}
|
||||
|
||||
void withFramedSink(std::function<void(Sink & sink)> fun);
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -320,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
|
|||
return res;
|
||||
} else {
|
||||
conn->to << wopQueryValidPaths;
|
||||
writeStorePaths(*this, conn->to, paths);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
conn.processStderr();
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -332,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryAllValidPaths;
|
||||
conn.processStderr();
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -349,9 +358,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
|
|||
return res;
|
||||
} else {
|
||||
conn->to << wopQuerySubstitutablePaths;
|
||||
writeStorePaths(*this, conn->to, paths);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
conn.processStderr();
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -373,7 +382,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info.deriver = parseStorePath(deriver);
|
||||
info.references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
info.downloadSize = readLongLong(conn->from);
|
||||
info.narSize = readLongLong(conn->from);
|
||||
infos.insert_or_assign(i.first, std::move(info));
|
||||
|
|
@ -386,9 +395,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
StorePathSet paths;
|
||||
for (auto & path : pathsMap)
|
||||
paths.insert(path.first);
|
||||
writeStorePaths(*this, conn->to, paths);
|
||||
worker_proto::write(*this, conn->to, paths);
|
||||
} else
|
||||
writeStorePathCAMap(*this, conn->to, pathsMap);
|
||||
worker_proto::write(*this, conn->to, pathsMap);
|
||||
conn.processStderr();
|
||||
size_t count = readNum<size_t>(conn->from);
|
||||
for (size_t n = 0; n < count; n++) {
|
||||
|
|
@ -396,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info.deriver = parseStorePath(deriver);
|
||||
info.references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
info.downloadSize = readLongLong(conn->from);
|
||||
info.narSize = readLongLong(conn->from);
|
||||
}
|
||||
|
|
@ -405,11 +414,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
}
|
||||
|
||||
|
||||
ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
|
||||
{
|
||||
auto deriver = readString(conn->from);
|
||||
auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
|
||||
auto info = make_ref<ValidPathInfo>(path, narHash);
|
||||
if (deriver != "") info->deriver = parseStorePath(deriver);
|
||||
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
conn->from >> info->registrationTime >> info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
|
||||
conn->from >> info->ultimate;
|
||||
info->sigs = readStrings<StringSet>(conn->from);
|
||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
||||
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
|
||||
{
|
||||
try {
|
||||
std::shared_ptr<ValidPathInfo> info;
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopQueryPathInfo << printStorePath(path);
|
||||
|
|
@ -425,17 +451,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
|||
bool valid; conn->from >> valid;
|
||||
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
||||
}
|
||||
auto deriver = readString(conn->from);
|
||||
auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
|
||||
info = std::make_shared<ValidPathInfo>(path, narHash);
|
||||
if (deriver != "") info->deriver = parseStorePath(deriver);
|
||||
info->references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
conn->from >> info->registrationTime >> info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
|
||||
conn->from >> info->ultimate;
|
||||
info->sigs = readStrings<StringSet>(conn->from);
|
||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
||||
}
|
||||
info = readValidPathInfo(conn, path);
|
||||
}
|
||||
callback(std::move(info));
|
||||
} catch (...) { callback.rethrow(); }
|
||||
|
|
@ -448,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryReferrers << printStorePath(path);
|
||||
conn.processStderr();
|
||||
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
|
||||
for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}))
|
||||
referrers.insert(i);
|
||||
}
|
||||
|
||||
|
|
@ -458,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
|
|||
auto conn(getConnection());
|
||||
conn->to << wopQueryValidDerivers << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -470,17 +486,32 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
|
|||
}
|
||||
conn->to << wopQueryDerivationOutputs << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return readStorePaths<StorePathSet>(*this, conn->from);
|
||||
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
|
||||
|
||||
if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
|
||||
auto conn(getConnection());
|
||||
conn->to << wopQueryDerivationOutputMap << printStorePath(path);
|
||||
conn.processStderr();
|
||||
return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {});
|
||||
} else {
|
||||
// Fallback for old daemon versions.
|
||||
// For floating-CA derivations (and their co-dependencies) this is an
|
||||
// under-approximation as it only returns the paths that can be inferred
|
||||
// from the derivation itself (and not the ones that are known because
|
||||
// the have been built), but as old stores don't handle floating-CA
|
||||
// derivations this shouldn't matter
|
||||
auto derivation = readDerivation(path);
|
||||
auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
|
||||
std::map<std::string, std::optional<StorePath>> ret;
|
||||
for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
|
||||
ret.emplace(outputName, outputAndPath.second);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
|
||||
|
|
@ -494,6 +525,93 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
|
|||
}
|
||||
|
||||
|
||||
ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
||||
Source & dump,
|
||||
const string & name,
|
||||
ContentAddressMethod caMethod,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair)
|
||||
{
|
||||
std::optional<ConnectionHandle> conn_(getConnection());
|
||||
auto & conn = *conn_;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
|
||||
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< renderContentAddressMethod(caMethod);
|
||||
worker_proto::write(*this, conn->to, references);
|
||||
conn->to << repair;
|
||||
|
||||
conn.withFramedSink([&](Sink & sink) {
|
||||
dump.drainInto(sink);
|
||||
});
|
||||
|
||||
auto path = parseStorePath(readString(conn->from));
|
||||
return readValidPathInfo(conn, path);
|
||||
}
|
||||
else {
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
|
||||
|
||||
std::visit(overloaded {
|
||||
[&](TextHashMethod thm) -> void {
|
||||
std::string s = dump.drain();
|
||||
conn->to << wopAddTextToStore << name << s;
|
||||
worker_proto::write(*this, conn->to, references);
|
||||
conn.processStderr();
|
||||
},
|
||||
[&](FixedOutputHashMethod fohm) -> void {
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(fohm.hashType);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
conn->to.warn = true;
|
||||
connections->incCapacity();
|
||||
{
|
||||
Finally cleanup([&]() { connections->decCapacity(); });
|
||||
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
|
||||
dump.drainInto(conn->to);
|
||||
} else {
|
||||
std::string contents = dump.drain();
|
||||
dumpString(contents, conn->to);
|
||||
}
|
||||
}
|
||||
conn->to.warn = false;
|
||||
conn.processStderr();
|
||||
} catch (SysError & e) {
|
||||
/* Daemon closed while we were sending the path. Probably OOM
|
||||
or I/O error. */
|
||||
if (e.errNo == EPIPE)
|
||||
try {
|
||||
conn.processStderr();
|
||||
} catch (EndOfFile & e) { }
|
||||
throw;
|
||||
}
|
||||
|
||||
}
|
||||
}, caMethod);
|
||||
auto path = parseStorePath(readString(conn->from));
|
||||
// Release our connection to prevent a deadlock in queryPathInfo().
|
||||
conn_.reset();
|
||||
return queryPathInfo(path);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method, HashType hashType, RepairFlag repair)
|
||||
{
|
||||
StorePathSet references;
|
||||
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
|
|
@ -509,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
sink
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
writeStorePaths(*this, sink, info.references);
|
||||
worker_proto::write(*this, sink, info.references);
|
||||
sink
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0 // == no legacy signature
|
||||
|
|
@ -519,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
|
||||
conn.processStderr(0, source2.get());
|
||||
|
||||
auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
assert(importedPaths.size() <= 1);
|
||||
}
|
||||
|
||||
|
|
@ -528,84 +646,15 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
writeStorePaths(*this, conn->to, info.references);
|
||||
worker_proto::write(*this, conn->to, info.references);
|
||||
conn->to << info.registrationTime << info.narSize
|
||||
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
|
||||
<< repair << !checkSigs;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
|
||||
|
||||
conn->to.flush();
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
struct FramedSink : BufferedSink
|
||||
{
|
||||
ConnectionHandle & conn;
|
||||
std::exception_ptr & ex;
|
||||
|
||||
FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
|
||||
{ }
|
||||
|
||||
~FramedSink()
|
||||
{
|
||||
try {
|
||||
conn->to << 0;
|
||||
conn->to.flush();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
void write(const unsigned char * data, size_t len) override
|
||||
{
|
||||
/* Don't send more data if the remote has
|
||||
encountered an error. */
|
||||
if (ex) {
|
||||
auto ex2 = ex;
|
||||
ex = nullptr;
|
||||
std::rethrow_exception(ex2);
|
||||
}
|
||||
conn->to << len;
|
||||
conn->to(data, len);
|
||||
};
|
||||
};
|
||||
|
||||
/* Handle log messages / exceptions from the remote on a
|
||||
separate thread. */
|
||||
std::thread stderrThread([&]()
|
||||
{
|
||||
try {
|
||||
conn.processStderr(nullptr, nullptr, false);
|
||||
} catch (...) {
|
||||
ex = std::current_exception();
|
||||
}
|
||||
});
|
||||
|
||||
Finally joinStderrThread([&]()
|
||||
{
|
||||
if (stderrThread.joinable()) {
|
||||
stderrThread.join();
|
||||
if (ex) {
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
FramedSink sink(conn, ex);
|
||||
conn.withFramedSink([&](Sink & sink) {
|
||||
copyNAR(source, sink);
|
||||
sink.flush();
|
||||
}
|
||||
|
||||
stderrThread.join();
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
|
||||
});
|
||||
} else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
|
||||
conn.processStderr(0, &source);
|
||||
} else {
|
||||
|
|
@ -616,57 +665,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
||||
|
||||
auto conn(getConnection());
|
||||
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (method == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(hashAlgo);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
conn->to.warn = true;
|
||||
connections->incCapacity();
|
||||
{
|
||||
Finally cleanup([&]() { connections->decCapacity(); });
|
||||
dumpPath(srcPath, conn->to, filter);
|
||||
}
|
||||
conn->to.warn = false;
|
||||
conn.processStderr();
|
||||
} catch (SysError & e) {
|
||||
/* Daemon closed while we were sending the path. Probably OOM
|
||||
or I/O error. */
|
||||
if (e.errNo == EPIPE)
|
||||
try {
|
||||
conn.processStderr();
|
||||
} catch (EndOfFile & e) { }
|
||||
throw;
|
||||
}
|
||||
|
||||
return parseStorePath(readString(conn->from));
|
||||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair)
|
||||
{
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
||||
|
||||
auto conn(getConnection());
|
||||
conn->to << wopAddTextToStore << name << s;
|
||||
writeStorePaths(*this, conn->to, references);
|
||||
|
||||
conn.processStderr();
|
||||
return parseStorePath(readString(conn->from));
|
||||
StringSource source(s);
|
||||
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -765,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
|
||||
conn->to
|
||||
<< wopCollectGarbage << options.action;
|
||||
writeStorePaths(*this, conn->to, options.pathsToDelete);
|
||||
worker_proto::write(*this, conn->to, options.pathsToDelete);
|
||||
conn->to << options.ignoreLiveness
|
||||
<< options.maxFreed
|
||||
/* removed options */
|
||||
|
|
@ -827,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
|
|||
ss.push_back(p.to_string(*this));
|
||||
conn->to << ss;
|
||||
conn.processStderr();
|
||||
willBuild = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
unknown = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
|
||||
conn->from >> downloadSize >> narSize;
|
||||
return;
|
||||
}
|
||||
|
|
@ -868,6 +871,18 @@ RemoteStore::Connection::~Connection()
|
|||
}
|
||||
}
|
||||
|
||||
void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
conn->to << wopNarFromPath << printStorePath(path);
|
||||
conn->processStderr();
|
||||
copyNAR(conn->from, sink);
|
||||
}
|
||||
|
||||
ref<FSAccessor> RemoteStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
||||
}
|
||||
|
||||
static Logger::Fields readFields(Source & from)
|
||||
{
|
||||
|
|
@ -910,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
|||
}
|
||||
|
||||
else if (msg == STDERR_ERROR) {
|
||||
string error = readString(from);
|
||||
unsigned int status = readInt(from);
|
||||
return std::make_exception_ptr(Error(status, error));
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
|
||||
return std::make_exception_ptr(readError(from));
|
||||
} else {
|
||||
string error = readString(from);
|
||||
unsigned int status = readInt(from);
|
||||
return std::make_exception_ptr(Error(status, error));
|
||||
}
|
||||
}
|
||||
|
||||
else if (msg == STDERR_NEXT)
|
||||
|
|
@ -950,14 +969,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
static std::string uriScheme = "unix://";
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
|
||||
{
|
||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
|
||||
});
|
||||
(*this)->to.flush();
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
/* Handle log messages / exceptions from the remote on a
|
||||
separate thread. */
|
||||
std::thread stderrThread([&]()
|
||||
{
|
||||
try {
|
||||
processStderr(nullptr, nullptr, false);
|
||||
} catch (...) {
|
||||
ex = std::current_exception();
|
||||
}
|
||||
});
|
||||
|
||||
Finally joinStderrThread([&]()
|
||||
{
|
||||
if (stderrThread.joinable()) {
|
||||
stderrThread.join();
|
||||
if (ex) {
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
FramedSink sink((*this)->to, ex);
|
||||
fun(sink);
|
||||
sink.flush();
|
||||
}
|
||||
|
||||
stderrThread.join();
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,19 +16,23 @@ struct FdSource;
|
|||
template<typename T> class Pool;
|
||||
struct ConnectionHandle;
|
||||
|
||||
struct RemoteStoreConfig : virtual StoreConfig
|
||||
{
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
const Setting<int> maxConnections{(StoreConfig*) this, 1,
|
||||
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
|
||||
|
||||
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
|
||||
"max-connection-age", "number of seconds to reuse a connection"};
|
||||
};
|
||||
|
||||
/* FIXME: RemoteStore is a misnomer - should be something like
|
||||
DaemonStore. */
|
||||
class RemoteStore : public virtual Store
|
||||
class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const Setting<int> maxConnections{(Store*) this, 1,
|
||||
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
|
||||
|
||||
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
|
||||
"max-connection-age", "number of seconds to reuse a connection"};
|
||||
|
||||
virtual bool sameMachine() = 0;
|
||||
|
||||
RemoteStore(const Params & params);
|
||||
|
|
@ -59,13 +63,21 @@ public:
|
|||
void querySubstitutablePathInfos(const StorePathCAMap & paths,
|
||||
SubstitutablePathInfos & infos) override;
|
||||
|
||||
/* Add a content-addressable store path. `dump` will be drained. */
|
||||
ref<const ValidPathInfo> addCAToStore(
|
||||
Source & dump,
|
||||
const string & name,
|
||||
ContentAddressMethod caMethod,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair);
|
||||
|
||||
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & nar,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
||||
|
|
@ -102,8 +114,6 @@ public:
|
|||
|
||||
void flushBadConnections();
|
||||
|
||||
protected:
|
||||
|
||||
struct Connection
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
|
|
@ -119,6 +129,8 @@ protected:
|
|||
|
||||
ref<Connection> openConnectionWrapper();
|
||||
|
||||
protected:
|
||||
|
||||
virtual ref<Connection> openConnection() = 0;
|
||||
|
||||
void initConnection(Connection & conn);
|
||||
|
|
@ -131,24 +143,56 @@ protected:
|
|||
|
||||
friend struct ConnectionHandle;
|
||||
|
||||
virtual ref<FSAccessor> getFSAccessor() override;
|
||||
|
||||
virtual void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
|
||||
ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
|
||||
|
||||
private:
|
||||
|
||||
std::atomic_bool failed{false};
|
||||
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore
|
||||
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
|
||||
{
|
||||
UDSRemoteStoreConfig(const Store::Params & params)
|
||||
: StoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
{
|
||||
}
|
||||
|
||||
UDSRemoteStoreConfig()
|
||||
: UDSRemoteStoreConfig(Store::Params({}))
|
||||
{
|
||||
}
|
||||
|
||||
const std::string name() override { return "Local Daemon Store"; }
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
UDSRemoteStore(const Params & params);
|
||||
UDSRemoteStore(std::string path, const Params & params);
|
||||
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
|
||||
|
||||
std::string getUri() override;
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{ return {"unix"}; }
|
||||
|
||||
bool sameMachine() override
|
||||
{ return true; }
|
||||
|
||||
ref<FSAccessor> getFSAccessor() override
|
||||
{ return LocalFSStore::getFSAccessor(); }
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{ LocalFSStore::narFromPath(path, sink); }
|
||||
|
||||
private:
|
||||
|
||||
ref<RemoteStore::Connection> openConnection() override;
|
||||
|
|
|
|||
|
|
@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
|
|||
return res;
|
||||
}
|
||||
|
||||
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
|
||||
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
||||
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
|
||||
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
|
||||
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
||||
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
|
||||
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
|
||||
const Setting<bool> multipartUpload{
|
||||
this, false, "multipart-upload", "whether to use multi-part uploads"};
|
||||
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
|
||||
const Setting<uint64_t> bufferSize{
|
||||
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
|
||||
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
|
||||
|
||||
const std::string name() override { return "S3 Binary Cache Store"; }
|
||||
};
|
||||
|
||||
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
|
||||
{
|
||||
std::string bucketName;
|
||||
|
||||
Stats stats;
|
||||
|
|
@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
S3Helper s3Helper;
|
||||
|
||||
S3BinaryCacheStoreImpl(
|
||||
const Params & params, const std::string & bucketName)
|
||||
: S3BinaryCacheStore(params)
|
||||
const std::string & scheme,
|
||||
const std::string & bucketName,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, S3BinaryCacheStore(params)
|
||||
, bucketName(bucketName)
|
||||
, s3Helper(profile, region, scheme, endpoint)
|
||||
{
|
||||
|
|
@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
return paths;
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() { return {"s3"}; }
|
||||
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, 5) != "s3://") return 0;
|
||||
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore;
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,19 +8,25 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "ssh-ng://";
|
||||
struct SSHStoreConfig : virtual RemoteStoreConfig
|
||||
{
|
||||
using RemoteStoreConfig::RemoteStoreConfig;
|
||||
|
||||
class SSHStore : public RemoteStore
|
||||
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
const std::string name() override { return "SSH Store"; }
|
||||
};
|
||||
|
||||
class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
SSHStore(const std::string & host, const Params & params)
|
||||
: Store(params)
|
||||
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, RemoteStore(params)
|
||||
, host(host)
|
||||
, master(
|
||||
|
|
@ -32,18 +38,16 @@ public:
|
|||
{
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
|
||||
|
||||
std::string getUri() override
|
||||
{
|
||||
return uriScheme + host;
|
||||
return *uriSchemes().begin() + "://" + host;
|
||||
}
|
||||
|
||||
bool sameMachine() override
|
||||
{ return false; }
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
|
||||
ref<FSAccessor> getFSAccessor() override;
|
||||
|
||||
private:
|
||||
|
||||
struct Connection : RemoteStore::Connection
|
||||
|
|
@ -68,19 +72,6 @@ private:
|
|||
};
|
||||
};
|
||||
|
||||
void SSHStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
conn->to << wopNarFromPath << printStorePath(path);
|
||||
conn->processStderr();
|
||||
copyNAR(conn->from, sink);
|
||||
}
|
||||
|
||||
ref<FSAccessor> SSHStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
||||
}
|
||||
|
||||
ref<RemoteStore::Connection> SSHStore::openConnection()
|
||||
{
|
||||
auto conn = make_ref<Connection>();
|
||||
|
|
@ -89,16 +80,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
|
|||
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
|
||||
conn->to = FdSink(conn->sshConn->in.get());
|
||||
conn->from = FdSource(conn->sshConn->out.get());
|
||||
initConnection(*conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
|
||||
});
|
||||
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@
|
|||
#include "json.hh"
|
||||
#include "url.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -140,21 +138,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
|
|||
*/
|
||||
|
||||
|
||||
StorePath Store::makeStorePath(const string & type,
|
||||
const Hash & hash, std::string_view name) const
|
||||
StorePath Store::makeStorePath(std::string_view type,
|
||||
std::string_view hash, std::string_view name) const
|
||||
{
|
||||
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
||||
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
|
||||
string s = std::string { type } + ":" + std::string { hash }
|
||||
+ ":" + storeDir + ":" + std::string { name };
|
||||
auto h = compressHash(hashString(htSHA256, s), 20);
|
||||
return StorePath(h, name);
|
||||
}
|
||||
|
||||
|
||||
StorePath Store::makeOutputPath(const string & id,
|
||||
StorePath Store::makeStorePath(std::string_view type,
|
||||
const Hash & hash, std::string_view name) const
|
||||
{
|
||||
return makeStorePath("output:" + id, hash,
|
||||
std::string(name) + (id == "out" ? "" : "-" + id));
|
||||
return makeStorePath(type, hash.to_string(Base16, true), name);
|
||||
}
|
||||
|
||||
|
||||
StorePath Store::makeOutputPath(std::string_view id,
|
||||
const Hash & hash, std::string_view name) const
|
||||
{
|
||||
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -339,7 +344,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
|||
|
||||
|
||||
Store::Store(const Params & params)
|
||||
: Config(params)
|
||||
: StoreConfig(params)
|
||||
, state({(size_t) pathInfoCacheSize})
|
||||
{
|
||||
}
|
||||
|
|
@ -1002,7 +1007,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1012,9 +1016,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
|||
|
||||
namespace nix {
|
||||
|
||||
|
||||
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
|
||||
|
||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
|
||||
{
|
||||
|
|
@ -1028,24 +1029,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
|||
return {uri, params};
|
||||
}
|
||||
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||
auto params = extraParams;
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
|
||||
for (auto fun : *RegisterStoreImplementation::implementations) {
|
||||
auto store = fun(uri, params);
|
||||
if (store) {
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
|
||||
throw Error("don't know how to open Nix store '%s'", uri);
|
||||
}
|
||||
|
||||
static bool isNonUriPath(const std::string & spec) {
|
||||
return
|
||||
// is not a URL
|
||||
|
|
@ -1055,44 +1038,62 @@ static bool isNonUriPath(const std::string & spec) {
|
|||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
||||
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
|
||||
{
|
||||
if (uri == "daemon") {
|
||||
return tDaemon;
|
||||
} else if (uri == "local" || isNonUriPath(uri)) {
|
||||
return tLocal;
|
||||
} else if (uri == "" || uri == "auto") {
|
||||
if (uri == "" || uri == "auto") {
|
||||
auto stateDir = get(params, "state").value_or(settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return tLocal;
|
||||
return std::make_shared<LocalStore>(params);
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return tDaemon;
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
else
|
||||
return tLocal;
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (uri == "daemon") {
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
} else if (uri == "local") {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (isNonUriPath(uri)) {
|
||||
Store::Params params2 = params;
|
||||
params2["root"] = absPath(uri);
|
||||
return std::make_shared<LocalStore>(params2);
|
||||
} else {
|
||||
return tOther;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
|
||||
case tDaemon:
|
||||
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
||||
case tLocal: {
|
||||
Store::Params params2 = params;
|
||||
if (isNonUriPath(uri)) {
|
||||
params2["root"] = absPath(uri);
|
||||
}
|
||||
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
||||
}
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
});
|
||||
auto params = extraParams;
|
||||
try {
|
||||
auto parsedUri = parseURL(uri_);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
|
||||
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
|
||||
|
||||
for (auto implem : *Implementations::registered) {
|
||||
if (implem.uriSchemes.count(parsedUri.scheme)) {
|
||||
auto store = implem.create(parsedUri.scheme, baseURI, params);
|
||||
if (store) {
|
||||
store->init();
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (BadURL &) {
|
||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
|
||||
if (auto store = openFromNonUri(uri, params)) {
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
|
||||
throw Error("don't know how to open Nix store '%s'", uri_);
|
||||
}
|
||||
|
||||
std::list<ref<Store>> getDefaultSubstituters()
|
||||
{
|
||||
|
|
@ -1126,5 +1127,6 @@ std::list<ref<Store>> getDefaultSubstituters()
|
|||
return stores;
|
||||
}
|
||||
|
||||
std::vector<StoreFactory> * Implementations::registered = 0;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,31 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* About the class hierarchy of the store implementations:
|
||||
*
|
||||
* Each store type `Foo` consists of two classes:
|
||||
*
|
||||
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
|
||||
* for the store
|
||||
*
|
||||
* It should only contain members of type `const Setting<T>` (or subclasses
|
||||
* of it) and inherit the constructors of `StoreConfig`
|
||||
* (`using StoreConfig::StoreConfig`).
|
||||
*
|
||||
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
|
||||
* implementation of the store.
|
||||
*
|
||||
* This class is expected to have a constructor `Foo(const Params & params)`
|
||||
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
|
||||
* `assertion failure` when trying to instantiate it).
|
||||
*
|
||||
* You can then register the new store using:
|
||||
*
|
||||
* ```
|
||||
* cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
|
||||
* ```
|
||||
*/
|
||||
|
||||
MakeError(SubstError, Error);
|
||||
MakeError(BuildError, Error); // denotes a permanent build failure
|
||||
|
|
@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error);
|
|||
MakeError(SubstituterDisabled, Error);
|
||||
MakeError(BadStorePath, Error);
|
||||
|
||||
MakeError(InvalidStoreURI, Error);
|
||||
|
||||
class FSAccessor;
|
||||
class NarInfoDiskCache;
|
||||
|
|
@ -144,12 +170,33 @@ struct BuildResult
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
class Store : public std::enable_shared_from_this<Store>, public Config
|
||||
struct StoreConfig : public Config
|
||||
{
|
||||
public:
|
||||
using Config::Config;
|
||||
|
||||
typedef std::map<std::string, std::string> Params;
|
||||
/**
|
||||
* When constructing a store implementation, we pass in a map `params` of
|
||||
* parameters that's supposed to initialize the associated config.
|
||||
* To do that, we must use the `StoreConfig(StringMap & params)`
|
||||
* constructor, so we'd like to `delete` its default constructor to enforce
|
||||
* it.
|
||||
*
|
||||
* However, actually deleting it means that all the subclasses of
|
||||
* `StoreConfig` will have their default constructor deleted (because it's
|
||||
* supposed to call the deleted default constructor of `StoreConfig`). But
|
||||
* because we're always using virtual inheritance, the constructors of
|
||||
* child classes will never implicitely call this one, so deleting it will
|
||||
* be more painful than anything else.
|
||||
*
|
||||
* So we `assert(false)` here to ensure at runtime that the right
|
||||
* constructor is always called without having to redefine a custom
|
||||
* constructor for each `*Config` class.
|
||||
*/
|
||||
StoreConfig() { assert(false); }
|
||||
|
||||
virtual ~StoreConfig() { }
|
||||
|
||||
virtual const std::string name() = 0;
|
||||
|
||||
const PathSetting storeDir_{this, false, settings.nixStore,
|
||||
"store", "path to the Nix store"};
|
||||
|
|
@ -167,6 +214,14 @@ public:
|
|||
"system-features",
|
||||
"Optional features that the system this store builds on implements (like \"kvm\")."};
|
||||
|
||||
};
|
||||
|
||||
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
typedef std::map<std::string, std::string> Params;
|
||||
|
||||
protected:
|
||||
|
||||
struct PathInfoCacheValue {
|
||||
|
|
@ -200,6 +255,11 @@ protected:
|
|||
Store(const Params & params);
|
||||
|
||||
public:
|
||||
/**
|
||||
* Perform any necessary effectful operation to make the store up and
|
||||
* running
|
||||
*/
|
||||
virtual void init() {};
|
||||
|
||||
virtual ~Store() { }
|
||||
|
||||
|
|
@ -247,10 +307,12 @@ public:
|
|||
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
|
||||
|
||||
/* Constructs a unique store path name. */
|
||||
StorePath makeStorePath(const string & type,
|
||||
StorePath makeStorePath(std::string_view type,
|
||||
std::string_view hash, std::string_view name) const;
|
||||
StorePath makeStorePath(std::string_view type,
|
||||
const Hash & hash, std::string_view name) const;
|
||||
|
||||
StorePath makeOutputPath(const string & id,
|
||||
StorePath makeOutputPath(std::string_view id,
|
||||
const Hash & hash, std::string_view name) const;
|
||||
|
||||
StorePath makeFixedOutputPath(FileIngestionMethod method,
|
||||
|
|
@ -389,13 +451,12 @@ public:
|
|||
/* Like addToStore(), but the contents of the path are contained
|
||||
in `dump', which is either a NAR serialisation (if recursive ==
|
||||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
false).
|
||||
`dump` may be drained */
|
||||
// FIXME: remove?
|
||||
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
||||
{
|
||||
throw Error("addToStoreFromDump() is not supported by this store");
|
||||
}
|
||||
{ unsupported("addToStoreFromDump"); }
|
||||
|
||||
/* Like addToStore, but the contents written to the output path is
|
||||
a regular file containing the given string. */
|
||||
|
|
@ -418,8 +479,38 @@ public:
|
|||
BuildMode buildMode = bmNormal);
|
||||
|
||||
/* Build a single non-materialized derivation (i.e. not from an
|
||||
on-disk .drv file). Note that ‘drvPath’ is only used for
|
||||
informational purposes. */
|
||||
on-disk .drv file).
|
||||
|
||||
‘drvPath’ is used to deduplicate worker goals so it is imperative that
|
||||
is correct. That said, it doesn't literally need to be store path that
|
||||
would be calculated from writing this derivation to the store: it is OK
|
||||
if it instead is that of a Derivation which would resolve to this (by
|
||||
taking the outputs of it's input derivations and adding them as input
|
||||
sources) such that the build time referenceable-paths are the same.
|
||||
|
||||
In the input-addressed case, we usually *do* use an "original"
|
||||
unresolved derivations's path, as that is what will be used in the
|
||||
`buildPaths` case. Also, the input-addressed output paths are verified
|
||||
only by that contents of that specific unresolved derivation, so it is
|
||||
nice to keep that information around so if the original derivation is
|
||||
ever obtained later, it can be verified whether the trusted user in fact
|
||||
used the proper output path.
|
||||
|
||||
In the content-addressed case, we want to always use the
|
||||
resolved drv path calculated from the provided derivation. This serves
|
||||
two purposes:
|
||||
|
||||
- It keeps the operation trustless, by ruling out a maliciously
|
||||
invalid drv path corresponding to a non-resolution-equivalent
|
||||
derivation.
|
||||
|
||||
- For the floating case in particular, it ensures that the derivation
|
||||
to output mapping respects the resolution equivalence relation, so
|
||||
one cannot choose different resolution-equivalent derivations to
|
||||
subvert dependency coherence (i.e. the property that one doesn't end
|
||||
up with multiple different versions of dependencies without
|
||||
explicitly choosing to allow it).
|
||||
*/
|
||||
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode = bmNormal) = 0;
|
||||
|
||||
|
|
@ -456,7 +547,7 @@ public:
|
|||
- The collector isn't running, or it's just started but hasn't
|
||||
acquired the GC lock yet. In that case we get and release
|
||||
the lock right away, then exit. The collector scans the
|
||||
permanent root and sees our's.
|
||||
permanent root and sees ours.
|
||||
|
||||
In either case the permanent root is seen by the collector. */
|
||||
virtual void syncWithGC() { };
|
||||
|
|
@ -624,22 +715,25 @@ protected:
|
|||
|
||||
};
|
||||
|
||||
|
||||
class LocalFSStore : public virtual Store
|
||||
struct LocalFSStoreConfig : virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
// FIXME: the (Store*) cast works around a bug in gcc that causes
|
||||
using StoreConfig::StoreConfig;
|
||||
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
|
||||
// it to omit the call to the Setting constructor. Clang works fine
|
||||
// either way.
|
||||
const PathSetting rootDir{(Store*) this, true, "",
|
||||
const PathSetting rootDir{(StoreConfig*) this, true, "",
|
||||
"root", "directory prefixed to all other paths"};
|
||||
const PathSetting stateDir{(Store*) this, false,
|
||||
const PathSetting stateDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
|
||||
"state", "directory where Nix will store state"};
|
||||
const PathSetting logDir{(Store*) this, false,
|
||||
const PathSetting logDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
|
||||
"log", "directory where Nix will store state"};
|
||||
};
|
||||
|
||||
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const static string drvsLogDir;
|
||||
|
||||
|
|
@ -728,37 +822,48 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
|
|||
const Store::Params & extraParams = Store::Params());
|
||||
|
||||
|
||||
enum StoreType {
|
||||
tDaemon,
|
||||
tLocal,
|
||||
tOther
|
||||
};
|
||||
|
||||
|
||||
StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
|
||||
const std::string & stateDir = settings.nixStateDir);
|
||||
|
||||
/* Return the default substituter stores, defined by the
|
||||
‘substituters’ option and various legacy options. */
|
||||
std::list<ref<Store>> getDefaultSubstituters();
|
||||
|
||||
|
||||
/* Store implementation registration. */
|
||||
typedef std::function<std::shared_ptr<Store>(
|
||||
const std::string & uri, const Store::Params & params)> OpenStore;
|
||||
|
||||
struct RegisterStoreImplementation
|
||||
struct StoreFactory
|
||||
{
|
||||
typedef std::vector<OpenStore> Implementations;
|
||||
static Implementations * implementations;
|
||||
std::set<std::string> uriSchemes;
|
||||
std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
|
||||
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
|
||||
};
|
||||
|
||||
RegisterStoreImplementation(OpenStore fun)
|
||||
struct Implementations
|
||||
{
|
||||
static std::vector<StoreFactory> * registered;
|
||||
|
||||
template<typename T, typename TConfig>
|
||||
static void add()
|
||||
{
|
||||
if (!implementations) implementations = new Implementations;
|
||||
implementations->push_back(fun);
|
||||
if (!registered) registered = new std::vector<StoreFactory>();
|
||||
StoreFactory factory{
|
||||
.uriSchemes = T::uriSchemes(),
|
||||
.create =
|
||||
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{ return std::make_shared<T>(scheme, uri, params); }),
|
||||
.getConfig =
|
||||
([]()
|
||||
-> std::shared_ptr<StoreConfig>
|
||||
{ return std::make_shared<TConfig>(StringMap({})); })
|
||||
};
|
||||
registered->push_back(factory);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T, typename TConfig>
|
||||
struct RegisterStoreImplementation
|
||||
{
|
||||
RegisterStoreImplementation()
|
||||
{
|
||||
Implementations::add<T, TConfig>();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/* Display a set of paths in human-readable form (i.e., between quotes
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ namespace nix {
|
|||
#define WORKER_MAGIC_1 0x6e697863
|
||||
#define WORKER_MAGIC_2 0x6478696f
|
||||
|
||||
#define PROTOCOL_VERSION 0x118
|
||||
#define PROTOCOL_VERSION 0x11a
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ typedef enum {
|
|||
wopQueryReferences = 5, // obsolete
|
||||
wopQueryReferrers = 6,
|
||||
wopAddToStore = 7,
|
||||
wopAddTextToStore = 8,
|
||||
wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore
|
||||
wopBuildPaths = 9,
|
||||
wopEnsurePath = 10,
|
||||
wopAddTempRoot = 11,
|
||||
|
|
@ -66,10 +66,6 @@ typedef enum {
|
|||
class Store;
|
||||
struct Source;
|
||||
|
||||
template<class T> T readStorePaths(const Store & store, Source & from);
|
||||
|
||||
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
|
||||
|
||||
/* To guide overloading */
|
||||
template<typename T>
|
||||
struct Phantom {};
|
||||
|
|
@ -78,76 +74,81 @@ struct Phantom {};
|
|||
namespace worker_proto {
|
||||
/* FIXME maybe move more stuff inside here */
|
||||
|
||||
StorePath read(const Store & store, Source & from, Phantom<StorePath> _);
|
||||
void write(const Store & store, Sink & out, const StorePath & storePath);
|
||||
#define MAKE_WORKER_PROTO(TEMPLATE, T) \
|
||||
TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \
|
||||
TEMPLATE void write(const Store & store, Sink & out, const T & str)
|
||||
|
||||
template<typename T>
|
||||
std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _);
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::map<string, T> & resMap);
|
||||
template<typename T>
|
||||
std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _);
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::optional<T> & optVal);
|
||||
MAKE_WORKER_PROTO(, std::string);
|
||||
MAKE_WORKER_PROTO(, StorePath);
|
||||
MAKE_WORKER_PROTO(, ContentAddress);
|
||||
|
||||
/* Specialization which uses and empty string for the empty case, taking
|
||||
advantage of the fact StorePaths always serialize to a non-empty string.
|
||||
This is done primarily for backwards compatability, so that StorePath <=
|
||||
std::optional<StorePath>, where <= is the compatability partial order.
|
||||
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
|
||||
|
||||
#define X_ template<typename K, typename V>
|
||||
#define Y_ std::map<K, V>
|
||||
MAKE_WORKER_PROTO(X_, Y_);
|
||||
#undef X_
|
||||
#undef Y_
|
||||
|
||||
/* These use the empty string for the null case, relying on the fact
|
||||
that the underlying types never serialize to the empty string.
|
||||
|
||||
We do this instead of a generic std::optional<T> instance because
|
||||
ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
|
||||
the same reason, we don't have a std::variant<T..> instances (ordinal
|
||||
tags 0...n).
|
||||
|
||||
We could the generic instances and then these as specializations for
|
||||
compatability, but that's proven a bit finnicky, and also makes the
|
||||
worker protocol harder to implement in other languages where such
|
||||
specializations may not be allowed.
|
||||
*/
|
||||
template<>
|
||||
void write(const Store & store, Sink & out, const std::optional<StorePath> & optVal);
|
||||
MAKE_WORKER_PROTO(, std::optional<StorePath>);
|
||||
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
|
||||
|
||||
template<typename T>
|
||||
std::map<std::string, T> read(const Store & store, Source & from, Phantom<std::map<std::string, T>> _)
|
||||
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
|
||||
{
|
||||
std::map<string, T> resMap;
|
||||
auto size = (size_t)readInt(from);
|
||||
std::set<T> resSet;
|
||||
auto size = readNum<size_t>(from);
|
||||
while (size--) {
|
||||
auto thisKey = readString(from);
|
||||
resMap.insert_or_assign(std::move(thisKey), nix::worker_proto::read(store, from, Phantom<T> {}));
|
||||
resSet.insert(read(store, from, Phantom<T> {}));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::set<T> & resSet)
|
||||
{
|
||||
out << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
write(store, out, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _)
|
||||
{
|
||||
std::map<K, V> resMap;
|
||||
auto size = readNum<size_t>(from);
|
||||
while (size--) {
|
||||
auto k = read(store, from, Phantom<K> {});
|
||||
auto v = read(store, from, Phantom<V> {});
|
||||
resMap.insert_or_assign(std::move(k), std::move(v));
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
|
||||
template<typename K, typename V>
|
||||
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
|
||||
{
|
||||
out << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
out << i.first;
|
||||
nix::worker_proto::write(store, out, i.second);
|
||||
write(store, out, i.first);
|
||||
write(store, out, i.second);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
std::optional<T> read(const Store & store, Source & from, Phantom<std::optional<T>> _)
|
||||
{
|
||||
auto tag = readNum<uint8_t>(from);
|
||||
switch (tag) {
|
||||
case 0:
|
||||
return std::nullopt;
|
||||
case 1:
|
||||
return nix::worker_proto::read(store, from, Phantom<T> {});
|
||||
default:
|
||||
throw Error("got an invalid tag bit for std::optional: %#04x", tag);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write(const Store & store, Sink & out, const std::optional<T> & optVal)
|
||||
{
|
||||
out << (optVal ? 1 : 0);
|
||||
if (optVal)
|
||||
nix::worker_proto::write(store, out, *optVal);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
|
||||
|
||||
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue