mirror of
https://github.com/NixOS/nix.git
synced 2025-11-20 01:09:37 +01:00
Merge remote-tracking branch 'upstream/master' into remove-storetype-delegate-regStore
This commit is contained in:
John Ericson
commit
f60b380a7f
62 changed files with 1694 additions and 857 deletions
|
|
@ -22,7 +22,8 @@
|
|||
namespace nix {
|
||||
|
||||
BinaryCacheStore::BinaryCacheStore(const Params & params)
|
||||
: Store(params)
|
||||
: BinaryCacheStoreConfig(params)
|
||||
, Store(params)
|
||||
{
|
||||
if (secretKeyFile != "")
|
||||
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
|
||||
|
|
|
|||
|
|
@ -11,17 +11,21 @@ namespace nix {
|
|||
|
||||
struct NarInfo;
|
||||
|
||||
class BinaryCacheStore : public Store
|
||||
struct BinaryCacheStoreConfig : virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
||||
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
|
||||
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
||||
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
|
||||
const Setting<bool> parallelCompression{this, false, "parallel-compression",
|
||||
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
||||
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
|
||||
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
||||
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
|
||||
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
|
||||
"enable multi-threading compression, available for xz only currently"};
|
||||
};
|
||||
|
||||
class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
|
||||
{
|
||||
|
||||
private:
|
||||
|
||||
|
|
@ -58,7 +62,7 @@ public:
|
|||
|
||||
public:
|
||||
|
||||
virtual void init();
|
||||
virtual void init() override;
|
||||
|
||||
private:
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
|
||||
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
|
||||
|
|
@ -15,7 +15,7 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
|
|||
},
|
||||
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
|
||||
return {
|
||||
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
|
||||
dof.path(store, drvName, outputName)
|
||||
};
|
||||
},
|
||||
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
|
||||
|
|
@ -25,6 +25,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
|
|||
}
|
||||
|
||||
|
||||
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
|
||||
return store.makeFixedOutputPath(
|
||||
hash.method, hash.hash,
|
||||
outputPathName(drvName, outputName));
|
||||
}
|
||||
|
||||
|
||||
bool derivationIsCA(DerivationType dt) {
|
||||
switch (dt) {
|
||||
case DerivationType::InputAddressed: return false;
|
||||
|
|
@ -106,12 +113,15 @@ static string parseString(std::istream & str)
|
|||
return res;
|
||||
}
|
||||
|
||||
static void validatePath(std::string_view s) {
|
||||
if (s.size() == 0 || s[0] != '/')
|
||||
throw FormatError("bad path '%1%' in derivation", s);
|
||||
}
|
||||
|
||||
static Path parsePath(std::istream & str)
|
||||
{
|
||||
string s = parseString(str);
|
||||
if (s.size() == 0 || s[0] != '/')
|
||||
throw FormatError("bad path '%1%' in derivation", s);
|
||||
auto s = parseString(str);
|
||||
validatePath(s);
|
||||
return s;
|
||||
}
|
||||
|
||||
|
|
@ -140,7 +150,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
|
|||
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store,
|
||||
StorePath path, std::string_view hashAlgo, std::string_view hash)
|
||||
std::string_view pathS, std::string_view hashAlgo, std::string_view hash)
|
||||
{
|
||||
if (hashAlgo != "") {
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
|
|
@ -148,40 +158,45 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
|||
method = FileIngestionMethod::Recursive;
|
||||
hashAlgo = hashAlgo.substr(2);
|
||||
}
|
||||
const HashType hashType = parseHashType(hashAlgo);
|
||||
|
||||
return hash != ""
|
||||
? DerivationOutput {
|
||||
.output = DerivationOutputCAFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
||||
},
|
||||
}
|
||||
}
|
||||
: (settings.requireExperimentalFeature("ca-derivations"),
|
||||
DerivationOutput {
|
||||
.output = DerivationOutputCAFloating {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
},
|
||||
});
|
||||
} else
|
||||
const auto hashType = parseHashType(hashAlgo);
|
||||
if (hash != "") {
|
||||
validatePath(pathS);
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputCAFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
settings.requireExperimentalFeature("ca-derivations");
|
||||
assert(pathS == "");
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputCAFloating {
|
||||
.method = std::move(method),
|
||||
.hashType = std::move(hashType),
|
||||
},
|
||||
};
|
||||
}
|
||||
} else {
|
||||
validatePath(pathS);
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = std::move(path),
|
||||
.path = store.parseStorePath(pathS),
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
|
||||
{
|
||||
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
|
||||
expect(str, ","); const auto pathS = parseString(str);
|
||||
expect(str, ","); const auto hashAlgo = parseString(str);
|
||||
expect(str, ","); const auto hash = parseString(str);
|
||||
expect(str, ")");
|
||||
|
||||
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -294,17 +309,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
|
|||
for (auto & i : outputs) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, i.first);
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
|
||||
std::visit(overloaded {
|
||||
[&](DerivationOutputInputAddressed doi) {
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
[&](DerivationOutputCAFixed dof) {
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
|
||||
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
|
||||
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
||||
s += ','; printUnquotedString(s, "");
|
||||
},
|
||||
|
|
@ -360,6 +377,16 @@ bool isDerivation(const string & fileName)
|
|||
}
|
||||
|
||||
|
||||
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
|
||||
std::string res { drvName };
|
||||
if (outputName != "out") {
|
||||
res += "-";
|
||||
res += outputName;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
DerivationType BasicDerivation::type() const
|
||||
{
|
||||
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
|
||||
|
|
@ -452,12 +479,12 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
|||
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
|
||||
case DerivationType::CAFixed: {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & i : drv.outputsAndPaths(store)) {
|
||||
auto & dof = std::get<DerivationOutputCAFixed>(i.second.first.output);
|
||||
for (const auto & i : drv.outputs) {
|
||||
auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
|
||||
auto hash = hashString(htSHA256, "fixed:out:"
|
||||
+ dof.hash.printMethodAlgo() + ":"
|
||||
+ dof.hash.hash.to_string(Base16, false) + ":"
|
||||
+ store.printStorePath(i.second.second));
|
||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
return outputHashes;
|
||||
|
|
@ -508,21 +535,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
|
|||
}
|
||||
|
||||
|
||||
StorePathSet BasicDerivation::outputPaths(const Store & store) const
|
||||
{
|
||||
StorePathSet paths;
|
||||
for (auto & i : outputsAndPaths(store))
|
||||
paths.insert(i.second.second);
|
||||
return paths;
|
||||
}
|
||||
|
||||
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
||||
{
|
||||
auto path = store.parseStorePath(readString(in));
|
||||
const auto pathS = readString(in);
|
||||
const auto hashAlgo = readString(in);
|
||||
const auto hash = readString(in);
|
||||
|
||||
return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
|
||||
return parseDerivationOutput(store, pathS, hashAlgo, hash);
|
||||
}
|
||||
|
||||
StringSet BasicDerivation::outputNames() const
|
||||
|
|
@ -533,23 +552,12 @@ StringSet BasicDerivation::outputNames() const
|
|||
return names;
|
||||
}
|
||||
|
||||
DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
|
||||
DerivationOutputsAndPaths outsAndPaths;
|
||||
for (auto output : outputs)
|
||||
outsAndPaths.insert(std::make_pair(
|
||||
output.first,
|
||||
std::make_pair(output.second, output.second.path(store, name))
|
||||
)
|
||||
);
|
||||
return outsAndPaths;
|
||||
}
|
||||
|
||||
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
|
||||
DerivationOutputsAndOptPaths outsAndOptPaths;
|
||||
for (auto output : outputs)
|
||||
outsAndOptPaths.insert(std::make_pair(
|
||||
output.first,
|
||||
std::make_pair(output.second, output.second.pathOpt(store, output.first))
|
||||
std::make_pair(output.second, output.second.path(store, name, output.first))
|
||||
)
|
||||
);
|
||||
return outsAndOptPaths;
|
||||
|
|
@ -594,22 +602,25 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
|
|||
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
|
||||
{
|
||||
out << drv.outputs.size();
|
||||
for (auto & i : drv.outputsAndPaths(store)) {
|
||||
out << i.first
|
||||
<< store.printStorePath(i.second.second);
|
||||
for (auto & i : drv.outputs) {
|
||||
out << i.first;
|
||||
std::visit(overloaded {
|
||||
[&](DerivationOutputInputAddressed doi) {
|
||||
out << "" << "";
|
||||
out << store.printStorePath(doi.path)
|
||||
<< ""
|
||||
<< "";
|
||||
},
|
||||
[&](DerivationOutputCAFixed dof) {
|
||||
out << dof.hash.printMethodAlgo()
|
||||
out << store.printStorePath(dof.path(store, drv.name, i.first))
|
||||
<< dof.hash.printMethodAlgo()
|
||||
<< dof.hash.hash.to_string(Base16, false);
|
||||
},
|
||||
[&](DerivationOutputCAFloating dof) {
|
||||
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
||||
out << ""
|
||||
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
|
||||
<< "";
|
||||
},
|
||||
}, i.second.first.output);
|
||||
}, i.second.output);
|
||||
}
|
||||
writeStorePaths(store, out, drv.inputSrcs);
|
||||
out << drv.platform << drv.builder << drv.args;
|
||||
|
|
@ -625,5 +636,12 @@ std::string hashPlaceholder(const std::string & outputName)
|
|||
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
|
||||
}
|
||||
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
|
||||
{
|
||||
auto drvNameWithExtension = drvPath.name();
|
||||
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
|
||||
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
|
||||
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
|
|||
struct DerivationOutputCAFixed
|
||||
{
|
||||
FixedOutputHash hash; /* hash used for expected hash computation */
|
||||
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
|
||||
};
|
||||
|
||||
/* Floating-output derivations, whose output paths are content addressed, but
|
||||
|
|
@ -49,14 +50,8 @@ struct DerivationOutput
|
|||
std::optional<HashType> hashAlgoOpt(const Store & store) const;
|
||||
/* Note, when you use this function you should make sure that you're passing
|
||||
the right derivation name. When in doubt, you should use the safer
|
||||
interface provided by BasicDerivation::outputsAndPaths */
|
||||
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
|
||||
/* DEPRECATED: Remove after CA drvs are fully implemented */
|
||||
StorePath path(const Store & store, std::string_view drvName) const {
|
||||
auto p = pathOpt(store, drvName);
|
||||
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
|
||||
return *p;
|
||||
}
|
||||
interface provided by BasicDerivation::outputsAndOptPaths */
|
||||
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
|
||||
};
|
||||
|
||||
typedef std::map<string, DerivationOutput> DerivationOutputs;
|
||||
|
|
@ -113,17 +108,12 @@ struct BasicDerivation
|
|||
/* Return true iff this is a fixed-output derivation. */
|
||||
DerivationType type() const;
|
||||
|
||||
/* Return the output paths of a derivation. */
|
||||
StorePathSet outputPaths(const Store & store) const;
|
||||
|
||||
/* Return the output names of a derivation. */
|
||||
StringSet outputNames() const;
|
||||
|
||||
/* Calculates the maps that contains all the DerivationOutputs, but
|
||||
augmented with knowledge of the Store paths they would be written into.
|
||||
The first one of these functions will be removed when the CA work is
|
||||
completed */
|
||||
DerivationOutputsAndPaths outputsAndPaths(const Store & store) const;
|
||||
augmented with knowledge of the Store paths they would be written
|
||||
into. */
|
||||
DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
|
||||
|
||||
static std::string_view nameFromPath(const StorePath & storePath);
|
||||
|
|
@ -155,6 +145,13 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
|
|||
// FIXME: remove
|
||||
bool isDerivation(const string & fileName);
|
||||
|
||||
/* Calculate the name that will be used for the store path for this
|
||||
output.
|
||||
|
||||
This is usually <drv-name>-<output-name>, but is just <drv-name> when
|
||||
the output name is "out". */
|
||||
std::string outputPathName(std::string_view drvName, std::string_view outputName);
|
||||
|
||||
// known CA drv's output hashes, current just for fixed-output derivations
|
||||
// whose output hashes are always known since they are fixed up-front.
|
||||
typedef std::map<std::string, Hash> CaOutputHashes;
|
||||
|
|
@ -202,6 +199,21 @@ struct Sink;
|
|||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
|
||||
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
|
||||
|
||||
/* This creates an opaque and almost certainly unique string
|
||||
deterministically from the output name.
|
||||
|
||||
It is used as a placeholder to allow derivations to refer to their
|
||||
own outputs without needing to use the hash of a derivation in
|
||||
itself, making the hash near-impossible to calculate. */
|
||||
std::string hashPlaceholder(const std::string & outputName);
|
||||
|
||||
/* This creates an opaque and almost certainly unique string
|
||||
deterministically from a derivation path and output name.
|
||||
|
||||
It is used as a placeholder to allow derivations to refer to
|
||||
content-addressed paths whose content --- and thus the path
|
||||
themselves --- isn't yet known. This occurs when a derivation has a
|
||||
dependency which is a CA derivation. */
|
||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,17 +2,27 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "dummy://";
|
||||
struct DummyStoreConfig : virtual StoreConfig {
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
struct DummyStore : public Store
|
||||
const std::string name() override { return "Dummy Store"; }
|
||||
};
|
||||
|
||||
struct DummyStore : public Store, public virtual DummyStoreConfig
|
||||
{
|
||||
DummyStore(const Params & params)
|
||||
: Store(params)
|
||||
DummyStore(const std::string scheme, const std::string uri, const Params & params)
|
||||
: DummyStore(params)
|
||||
{ }
|
||||
|
||||
DummyStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
{
|
||||
}
|
||||
|
||||
string getUri() override
|
||||
{
|
||||
return uriScheme;
|
||||
return *uriSchemes().begin();
|
||||
}
|
||||
|
||||
void queryPathInfoUncached(const StorePath & path,
|
||||
|
|
@ -21,6 +31,10 @@ struct DummyStore : public Store
|
|||
callback(nullptr);
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() {
|
||||
return {"dummy"};
|
||||
}
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
||||
{ unsupported("queryPathFromHashPart"); }
|
||||
|
||||
|
|
@ -48,12 +62,6 @@ struct DummyStore : public Store
|
|||
{ unsupported("buildDerivation"); }
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (uri != uriScheme) return nullptr;
|
||||
return std::make_shared<DummyStore>(params);
|
||||
});
|
||||
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,11 +162,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
|
|||
else abort();
|
||||
}
|
||||
|
||||
template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
|
||||
{
|
||||
return AbstractSetting::toJSON();
|
||||
}
|
||||
|
||||
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
|
||||
{
|
||||
args.addFlag({
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "config.hh"
|
||||
#include "abstractsettingtojson.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <map>
|
||||
|
|
|
|||
|
|
@ -7,7 +7,14 @@ namespace nix {
|
|||
|
||||
MakeError(UploadToHTTP, Error);
|
||||
|
||||
class HttpBinaryCacheStore : public BinaryCacheStore
|
||||
struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
|
||||
const std::string name() override { return "Http Binary Cache Store"; }
|
||||
};
|
||||
|
||||
class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -24,9 +31,12 @@ private:
|
|||
public:
|
||||
|
||||
HttpBinaryCacheStore(
|
||||
const Params & params, const Path & _cacheUri)
|
||||
: BinaryCacheStore(params)
|
||||
, cacheUri(_cacheUri)
|
||||
const std::string & scheme,
|
||||
const Path & _cacheUri,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStore(params)
|
||||
, cacheUri(scheme + "://" + _cacheUri)
|
||||
{
|
||||
if (cacheUri.back() == '/')
|
||||
cacheUri.pop_back();
|
||||
|
|
@ -55,6 +65,13 @@ public:
|
|||
}
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
|
||||
auto ret = std::set<std::string>({"http", "https"});
|
||||
if (forceHttp) ret.insert("file");
|
||||
return ret;
|
||||
}
|
||||
protected:
|
||||
|
||||
void maybeDisable()
|
||||
|
|
@ -162,18 +179,6 @@ protected:
|
|||
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
|
||||
if (std::string(uri, 0, 7) != "http://" &&
|
||||
std::string(uri, 0, 8) != "https://" &&
|
||||
(!forceHttp || std::string(uri, 0, 7) != "file://"))
|
||||
return 0;
|
||||
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,18 +9,24 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "ssh://";
|
||||
|
||||
struct LegacySSHStore : public Store
|
||||
struct LegacySSHStoreConfig : virtual StoreConfig
|
||||
{
|
||||
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
|
||||
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
|
||||
using StoreConfig::StoreConfig;
|
||||
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
|
||||
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
const std::string name() override { return "Legacy SSH Store"; }
|
||||
};
|
||||
|
||||
struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
|
||||
{
|
||||
// Hack for getting remote build log output.
|
||||
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
|
||||
// the documentation
|
||||
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
|
||||
struct Connection
|
||||
{
|
||||
|
|
@ -37,8 +43,11 @@ struct LegacySSHStore : public Store
|
|||
|
||||
SSHMaster master;
|
||||
|
||||
LegacySSHStore(const string & host, const Params & params)
|
||||
: Store(params)
|
||||
static std::set<std::string> uriSchemes() { return {"ssh"}; }
|
||||
|
||||
LegacySSHStore(const string & scheme, const string & host, const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, host(host)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
|
|
@ -84,7 +93,7 @@ struct LegacySSHStore : public Store
|
|||
|
||||
string getUri() override
|
||||
{
|
||||
return uriScheme + host;
|
||||
return *uriSchemes().begin() + "://" + host;
|
||||
}
|
||||
|
||||
void queryPathInfoUncached(const StorePath & path,
|
||||
|
|
@ -325,12 +334,6 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
|
||||
});
|
||||
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,14 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
class LocalBinaryCacheStore : public BinaryCacheStore
|
||||
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
|
||||
const std::string name() override { return "Local Binary Cache Store"; }
|
||||
};
|
||||
|
||||
class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -13,8 +20,11 @@ private:
|
|||
public:
|
||||
|
||||
LocalBinaryCacheStore(
|
||||
const Params & params, const Path & binaryCacheDir)
|
||||
: BinaryCacheStore(params)
|
||||
const std::string scheme,
|
||||
const Path & binaryCacheDir,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStore(params)
|
||||
, binaryCacheDir(binaryCacheDir)
|
||||
{
|
||||
}
|
||||
|
|
@ -26,6 +36,8 @@ public:
|
|||
return "file://" + binaryCacheDir;
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes();
|
||||
|
||||
protected:
|
||||
|
||||
bool fileExists(const std::string & path) override;
|
||||
|
|
@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
|
|||
return pathExists(binaryCacheDir + "/" + path);
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
std::set<std::string> LocalBinaryCacheStore::uriSchemes()
|
||||
{
|
||||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
|
||||
std::string(uri, 0, 7) != "file://")
|
||||
return 0;
|
||||
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
|
||||
return {};
|
||||
else
|
||||
return {"file"};
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,8 @@ namespace nix {
|
|||
|
||||
|
||||
LocalStore::LocalStore(const Params & params)
|
||||
: Store(params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
|
||||
"physical path to the Nix store"}
|
||||
|
|
@ -578,13 +579,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
|||
envHasRightPath(path, i.first);
|
||||
},
|
||||
[&](DerivationOutputCAFloating _) {
|
||||
throw UnimplementedError("floating CA output derivations are not yet implemented");
|
||||
/* Nothing to check */
|
||||
},
|
||||
}, i.second.output);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
|
||||
{
|
||||
auto state(_state.lock());
|
||||
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
|
||||
}
|
||||
|
||||
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(deriver)
|
||||
(outputName)
|
||||
(printStorePath(output))
|
||||
.exec();
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
uint64_t LocalStore::addValidPath(State & state,
|
||||
const ValidPathInfo & info, bool checkOutputs)
|
||||
{
|
||||
|
|
@ -618,12 +638,11 @@ uint64_t LocalStore::addValidPath(State & state,
|
|||
registration above is undone. */
|
||||
if (checkOutputs) checkDerivationOutputs(info.path, drv);
|
||||
|
||||
for (auto & i : drv.outputsAndPaths(*this)) {
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(id)
|
||||
(i.first)
|
||||
(printStorePath(i.second.second))
|
||||
.exec();
|
||||
for (auto & i : drv.outputsAndOptPaths(*this)) {
|
||||
/* Floating CA derivations have indeterminate output paths until
|
||||
they are built, so don't register anything in that case */
|
||||
if (i.second.second)
|
||||
linkDeriverToPath(state, id, i.first, *i.second.second);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1533,27 +1552,5 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
|
|||
}
|
||||
}
|
||||
|
||||
static bool isNonUriPath(const std::string & spec) {
|
||||
return
|
||||
// is not a URL
|
||||
spec.find("://") == std::string::npos
|
||||
// Has at least one path separator, and so isn't a single word that
|
||||
// might be special like "auto"
|
||||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
Store::Params params2 = params;
|
||||
if (uri == "local") {
|
||||
} else if (isNonUriPath(uri)) {
|
||||
params2["root"] = absPath(uri);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
||||
});
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,8 +30,19 @@ struct OptimiseStats
|
|||
uint64_t blocksFreed = 0;
|
||||
};
|
||||
|
||||
struct LocalStoreConfig : virtual LocalFSStoreConfig
|
||||
{
|
||||
using LocalFSStoreConfig::LocalFSStoreConfig;
|
||||
|
||||
class LocalStore : public LocalFSStore
|
||||
Setting<bool> requireSigs{(StoreConfig*) this,
|
||||
settings.requireSigs,
|
||||
"require-sigs", "whether store paths should have a trusted signature on import"};
|
||||
|
||||
const std::string name() override { return "Local Store"; }
|
||||
};
|
||||
|
||||
|
||||
class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
|
||||
{
|
||||
private:
|
||||
|
||||
|
|
@ -95,10 +106,6 @@ public:
|
|||
|
||||
private:
|
||||
|
||||
Setting<bool> requireSigs{(Store*) this,
|
||||
settings.requireSigs,
|
||||
"require-sigs", "whether store paths should have a trusted signature on import"};
|
||||
|
||||
const PublicKeys & getPublicKeys();
|
||||
|
||||
public:
|
||||
|
|
@ -279,6 +286,11 @@ private:
|
|||
specified by the ‘secret-key-files’ option. */
|
||||
void signPathInfo(ValidPathInfo & info);
|
||||
|
||||
/* Register the store path 'output' as the output named 'outputName' of
|
||||
derivation 'deriver'. */
|
||||
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
|
||||
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
|
||||
|
||||
Path getRealStoreDir() override { return realStoreDir; }
|
||||
|
||||
void createUser(const std::string & userName, uid_t userId) override;
|
||||
|
|
|
|||
|
|
@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
|||
return;
|
||||
}
|
||||
|
||||
PathSet invalid;
|
||||
/* true for regular derivations, and CA derivations for which we
|
||||
have a trust mapping for all wanted outputs. */
|
||||
auto knownOutputPaths = true;
|
||||
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
|
||||
if (!pathOpt) {
|
||||
knownOutputPaths = false;
|
||||
break;
|
||||
}
|
||||
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
|
||||
invalid.insert(printStorePath(*pathOpt));
|
||||
}
|
||||
if (knownOutputPaths && invalid.empty()) return;
|
||||
|
||||
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
|
||||
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
|
||||
|
||||
PathSet invalid;
|
||||
for (auto & j : drv->outputsAndPaths(*this))
|
||||
if (wantOutput(j.first, path.outputs)
|
||||
&& !isValidPath(j.second.second))
|
||||
invalid.insert(printStorePath(j.second.second));
|
||||
if (invalid.empty()) return;
|
||||
|
||||
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
|
||||
for (auto & output : invalid)
|
||||
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
|
||||
|
|
|
|||
|
|
@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
|
|||
std::pair<PathSet, HashResult> scanForReferences(const string & path,
|
||||
const PathSet & refs)
|
||||
{
|
||||
RefScanSink refsSink;
|
||||
HashSink hashSink { htSHA256 };
|
||||
TeeSink sink { refsSink, hashSink };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<PathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
PathSet scanForReferences(Sink & toTee,
|
||||
const string & path, const PathSet & refs)
|
||||
{
|
||||
RefScanSink refsSink;
|
||||
TeeSink sink { refsSink, toTee };
|
||||
std::map<string, Path> backMap;
|
||||
|
||||
/* For efficiency (and a higher hit rate), just search for the
|
||||
|
|
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
|
|||
found.insert(j->second);
|
||||
}
|
||||
|
||||
auto hash = hashSink.finish();
|
||||
|
||||
return std::pair<PathSet, HashResult>(found, hash);
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ namespace nix {
|
|||
|
||||
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
|
||||
|
||||
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
|
||||
|
||||
struct RewritingSink : Sink
|
||||
{
|
||||
std::string from, to, prev;
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
|
|||
/* TODO: Separate these store impls into different files, give them better names */
|
||||
RemoteStore::RemoteStore(const Params & params)
|
||||
: Store(params)
|
||||
, RemoteStoreConfig(params)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
[this]() {
|
||||
|
|
@ -132,19 +133,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
|
|||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(const Params & params)
|
||||
: Store(params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
|
||||
: Store(params)
|
||||
, LocalFSStore(params)
|
||||
, RemoteStore(params)
|
||||
, path(socket_path)
|
||||
UDSRemoteStore::UDSRemoteStore(
|
||||
const std::string scheme,
|
||||
std::string socket_path,
|
||||
const Params & params)
|
||||
: UDSRemoteStore(params)
|
||||
{
|
||||
path.emplace(socket_path);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -989,18 +992,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
static std::string_view uriScheme = "unix://";
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (hasPrefix(uri, uriScheme))
|
||||
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
|
||||
else if (uri == "daemon")
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
else
|
||||
return nullptr;
|
||||
});
|
||||
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,19 +16,23 @@ struct FdSource;
|
|||
template<typename T> class Pool;
|
||||
struct ConnectionHandle;
|
||||
|
||||
struct RemoteStoreConfig : virtual StoreConfig
|
||||
{
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
const Setting<int> maxConnections{(StoreConfig*) this, 1,
|
||||
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
|
||||
|
||||
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
|
||||
"max-connection-age", "number of seconds to reuse a connection"};
|
||||
};
|
||||
|
||||
/* FIXME: RemoteStore is a misnomer - should be something like
|
||||
DaemonStore. */
|
||||
class RemoteStore : public virtual Store
|
||||
class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const Setting<int> maxConnections{(Store*) this, 1,
|
||||
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
|
||||
|
||||
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
|
||||
"max-connection-age", "number of seconds to reuse a connection"};
|
||||
|
||||
virtual bool sameMachine() = 0;
|
||||
|
||||
RemoteStore(const Params & params);
|
||||
|
|
@ -141,15 +145,35 @@ private:
|
|||
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore
|
||||
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
|
||||
{
|
||||
UDSRemoteStoreConfig(const Store::Params & params)
|
||||
: StoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
{
|
||||
}
|
||||
|
||||
UDSRemoteStoreConfig()
|
||||
: UDSRemoteStoreConfig(Store::Params({}))
|
||||
{
|
||||
}
|
||||
|
||||
const std::string name() override { return "Local Daemon Store"; }
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
UDSRemoteStore(const Params & params);
|
||||
UDSRemoteStore(std::string path, const Params & params);
|
||||
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
|
||||
|
||||
std::string getUri() override;
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{ return {"unix"}; }
|
||||
|
||||
bool sameMachine() override
|
||||
{ return true; }
|
||||
|
||||
|
|
|
|||
|
|
@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
|
|||
return res;
|
||||
}
|
||||
|
||||
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
||||
{
|
||||
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
|
||||
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
||||
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
|
||||
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
|
||||
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
|
||||
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
|
||||
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
|
||||
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
|
||||
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
|
||||
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
|
||||
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
|
||||
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
|
||||
const Setting<bool> multipartUpload{
|
||||
this, false, "multipart-upload", "whether to use multi-part uploads"};
|
||||
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
|
||||
const Setting<uint64_t> bufferSize{
|
||||
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
|
||||
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
|
||||
|
||||
const std::string name() override { return "S3 Binary Cache Store"; }
|
||||
};
|
||||
|
||||
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
|
||||
{
|
||||
std::string bucketName;
|
||||
|
||||
Stats stats;
|
||||
|
|
@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
S3Helper s3Helper;
|
||||
|
||||
S3BinaryCacheStoreImpl(
|
||||
const Params & params, const std::string & bucketName)
|
||||
: S3BinaryCacheStore(params)
|
||||
const std::string & scheme,
|
||||
const std::string & bucketName,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, S3BinaryCacheStore(params)
|
||||
, bucketName(bucketName)
|
||||
, s3Helper(profile, region, scheme, endpoint)
|
||||
{
|
||||
|
|
@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
return paths;
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() { return {"s3"}; }
|
||||
|
||||
};
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, 5) != "s3://") return 0;
|
||||
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
|
||||
store->init();
|
||||
return store;
|
||||
});
|
||||
static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,19 +8,25 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
static std::string uriScheme = "ssh-ng://";
|
||||
struct SSHStoreConfig : virtual RemoteStoreConfig
|
||||
{
|
||||
using RemoteStoreConfig::RemoteStoreConfig;
|
||||
|
||||
class SSHStore : public RemoteStore
|
||||
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
const std::string name() override { return "SSH Store"; }
|
||||
};
|
||||
|
||||
class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
|
||||
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
|
||||
const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
|
||||
const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
|
||||
|
||||
SSHStore(const std::string & host, const Params & params)
|
||||
: Store(params)
|
||||
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||
: StoreConfig(params)
|
||||
, Store(params)
|
||||
, RemoteStore(params)
|
||||
, host(host)
|
||||
, master(
|
||||
|
|
@ -32,9 +38,11 @@ public:
|
|||
{
|
||||
}
|
||||
|
||||
static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
|
||||
|
||||
std::string getUri() override
|
||||
{
|
||||
return uriScheme + host;
|
||||
return *uriSchemes().begin() + "://" + host;
|
||||
}
|
||||
|
||||
bool sameMachine() override
|
||||
|
|
@ -75,12 +83,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
|
|||
return conn;
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{
|
||||
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
|
||||
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
|
||||
});
|
||||
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
|
|||
*/
|
||||
|
||||
|
||||
StorePath Store::makeStorePath(const string & type,
|
||||
const Hash & hash, std::string_view name) const
|
||||
StorePath Store::makeStorePath(std::string_view type,
|
||||
std::string_view hash, std::string_view name) const
|
||||
{
|
||||
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
|
||||
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
|
||||
string s = std::string { type } + ":" + std::string { hash }
|
||||
+ ":" + storeDir + ":" + std::string { name };
|
||||
auto h = compressHash(hashString(htSHA256, s), 20);
|
||||
return StorePath(h, name);
|
||||
}
|
||||
|
||||
|
||||
StorePath Store::makeOutputPath(const string & id,
|
||||
StorePath Store::makeStorePath(std::string_view type,
|
||||
const Hash & hash, std::string_view name) const
|
||||
{
|
||||
return makeStorePath("output:" + id, hash,
|
||||
std::string(name) + (id == "out" ? "" : "-" + id));
|
||||
return makeStorePath(type, hash.to_string(Base16, true), name);
|
||||
}
|
||||
|
||||
|
||||
StorePath Store::makeOutputPath(std::string_view id,
|
||||
const Hash & hash, std::string_view name) const
|
||||
{
|
||||
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -339,7 +346,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
|||
|
||||
|
||||
Store::Store(const Params & params)
|
||||
: Config(params)
|
||||
: StoreConfig(params)
|
||||
, state({(size_t) pathInfoCacheSize})
|
||||
{
|
||||
}
|
||||
|
|
@ -1002,7 +1009,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1012,9 +1018,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
|||
|
||||
namespace nix {
|
||||
|
||||
|
||||
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
|
||||
|
||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
|
||||
{
|
||||
|
|
@ -1028,44 +1031,71 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
|||
return {uri, params};
|
||||
}
|
||||
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||
auto params = extraParams;
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
|
||||
for (auto fun : *RegisterStoreImplementation::implementations) {
|
||||
auto store = fun(uri, params);
|
||||
if (store) {
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
|
||||
throw Error("don't know how to open Nix store '%s'", uri);
|
||||
static bool isNonUriPath(const std::string & spec) {
|
||||
return
|
||||
// is not a URL
|
||||
spec.find("://") == std::string::npos
|
||||
// Has at least one path separator, and so isn't a single word that
|
||||
// might be special like "auto"
|
||||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
|
||||
// Specific prefixes are handled by the specific types of store, while here we
|
||||
// handle the general cases not covered by the other ones.
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
|
||||
{
|
||||
auto stateDir = get(params, "state").value_or(settings.nixStateDir);
|
||||
if (uri == "" || uri == "auto") {
|
||||
if (uri == "" || uri == "auto") {
|
||||
auto stateDir = get(params, "state").value_or(settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return std::make_shared<LocalStore>(params);
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
else
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
});
|
||||
} else if (uri == "daemon") {
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
} else if (uri == "local") {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (isNonUriPath(uri)) {
|
||||
Store::Params params2 = params;
|
||||
params2["root"] = absPath(uri);
|
||||
return std::make_shared<LocalStore>(params2);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
auto params = extraParams;
|
||||
try {
|
||||
auto parsedUri = parseURL(uri_);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
|
||||
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
|
||||
|
||||
for (auto implem : *Implementations::registered) {
|
||||
if (implem.uriSchemes.count(parsedUri.scheme)) {
|
||||
auto store = implem.create(parsedUri.scheme, baseURI, params);
|
||||
if (store) {
|
||||
store->init();
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (BadURL &) {
|
||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
|
||||
if (auto store = openFromNonUri(uri, params)) {
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
|
||||
throw Error("don't know how to open Nix store '%s'", uri_);
|
||||
}
|
||||
|
||||
std::list<ref<Store>> getDefaultSubstituters()
|
||||
{
|
||||
|
|
@ -1099,5 +1129,6 @@ std::list<ref<Store>> getDefaultSubstituters()
|
|||
return stores;
|
||||
}
|
||||
|
||||
std::vector<StoreFactory> * Implementations::registered = 0;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,31 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* About the class hierarchy of the store implementations:
|
||||
*
|
||||
* Each store type `Foo` consists of two classes:
|
||||
*
|
||||
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
|
||||
* for the store
|
||||
*
|
||||
* It should only contain members of type `const Setting<T>` (or subclasses
|
||||
* of it) and inherit the constructors of `StoreConfig`
|
||||
* (`using StoreConfig::StoreConfig`).
|
||||
*
|
||||
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
|
||||
* implementation of the store.
|
||||
*
|
||||
* This class is expected to have a constructor `Foo(const Params & params)`
|
||||
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
|
||||
* `assertion failure` when trying to instantiate it).
|
||||
*
|
||||
* You can then register the new store using:
|
||||
*
|
||||
* ```
|
||||
* cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
|
||||
* ```
|
||||
*/
|
||||
|
||||
MakeError(SubstError, Error);
|
||||
MakeError(BuildError, Error); // denotes a permanent build failure
|
||||
|
|
@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error);
|
|||
MakeError(SubstituterDisabled, Error);
|
||||
MakeError(BadStorePath, Error);
|
||||
|
||||
MakeError(InvalidStoreURI, Error);
|
||||
|
||||
class FSAccessor;
|
||||
class NarInfoDiskCache;
|
||||
|
|
@ -144,12 +170,31 @@ struct BuildResult
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
class Store : public std::enable_shared_from_this<Store>, public Config
|
||||
struct StoreConfig : public Config
|
||||
{
|
||||
public:
|
||||
using Config::Config;
|
||||
|
||||
typedef std::map<std::string, std::string> Params;
|
||||
/**
|
||||
* When constructing a store implementation, we pass in a map `params` of
|
||||
* parameters that's supposed to initialize the associated config.
|
||||
* To do that, we must use the `StoreConfig(StringMap & params)`
|
||||
* constructor, so we'd like to `delete` its default constructor to enforce
|
||||
* it.
|
||||
*
|
||||
* However, actually deleting it means that all the subclasses of
|
||||
* `StoreConfig` will have their default constructor deleted (because it's
|
||||
* supposed to call the deleted default constructor of `StoreConfig`). But
|
||||
* because we're always using virtual inheritance, the constructors of
|
||||
* child classes will never implicitely call this one, so deleting it will
|
||||
* be more painful than anything else.
|
||||
*
|
||||
* So we `assert(false)` here to ensure at runtime that the right
|
||||
* constructor is always called without having to redefine a custom
|
||||
* constructor for each `*Config` class.
|
||||
*/
|
||||
StoreConfig() { assert(false); }
|
||||
|
||||
virtual const std::string name() = 0;
|
||||
|
||||
const PathSetting storeDir_{this, false, settings.nixStore,
|
||||
"store", "path to the Nix store"};
|
||||
|
|
@ -167,6 +212,14 @@ public:
|
|||
"system-features",
|
||||
"Optional features that the system this store builds on implements (like \"kvm\")."};
|
||||
|
||||
};
|
||||
|
||||
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
typedef std::map<std::string, std::string> Params;
|
||||
|
||||
protected:
|
||||
|
||||
struct PathInfoCacheValue {
|
||||
|
|
@ -200,6 +253,11 @@ protected:
|
|||
Store(const Params & params);
|
||||
|
||||
public:
|
||||
/**
|
||||
* Perform any necessary effectful operation to make the store up and
|
||||
* running
|
||||
*/
|
||||
virtual void init() {};
|
||||
|
||||
virtual ~Store() { }
|
||||
|
||||
|
|
@ -247,10 +305,12 @@ public:
|
|||
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
|
||||
|
||||
/* Constructs a unique store path name. */
|
||||
StorePath makeStorePath(const string & type,
|
||||
StorePath makeStorePath(std::string_view type,
|
||||
std::string_view hash, std::string_view name) const;
|
||||
StorePath makeStorePath(std::string_view type,
|
||||
const Hash & hash, std::string_view name) const;
|
||||
|
||||
StorePath makeOutputPath(const string & id,
|
||||
StorePath makeOutputPath(std::string_view id,
|
||||
const Hash & hash, std::string_view name) const;
|
||||
|
||||
StorePath makeFixedOutputPath(FileIngestionMethod method,
|
||||
|
|
@ -624,22 +684,25 @@ protected:
|
|||
|
||||
};
|
||||
|
||||
|
||||
class LocalFSStore : public virtual Store
|
||||
struct LocalFSStoreConfig : virtual StoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
// FIXME: the (Store*) cast works around a bug in gcc that causes
|
||||
using StoreConfig::StoreConfig;
|
||||
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
|
||||
// it to omit the call to the Setting constructor. Clang works fine
|
||||
// either way.
|
||||
const PathSetting rootDir{(Store*) this, true, "",
|
||||
const PathSetting rootDir{(StoreConfig*) this, true, "",
|
||||
"root", "directory prefixed to all other paths"};
|
||||
const PathSetting stateDir{(Store*) this, false,
|
||||
const PathSetting stateDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
|
||||
"state", "directory where Nix will store state"};
|
||||
const PathSetting logDir{(Store*) this, false,
|
||||
const PathSetting logDir{(StoreConfig*) this, false,
|
||||
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
|
||||
"log", "directory where Nix will store state"};
|
||||
};
|
||||
|
||||
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
|
||||
{
|
||||
public:
|
||||
|
||||
const static string drvsLogDir;
|
||||
|
||||
|
|
@ -732,23 +795,43 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
|
|||
‘substituters’ option and various legacy options. */
|
||||
std::list<ref<Store>> getDefaultSubstituters();
|
||||
|
||||
|
||||
/* Store implementation registration. */
|
||||
typedef std::function<std::shared_ptr<Store>(
|
||||
const std::string & uri, const Store::Params & params)> OpenStore;
|
||||
|
||||
struct RegisterStoreImplementation
|
||||
struct StoreFactory
|
||||
{
|
||||
typedef std::vector<OpenStore> Implementations;
|
||||
static Implementations * implementations;
|
||||
std::set<std::string> uriSchemes;
|
||||
std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
|
||||
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
|
||||
};
|
||||
struct Implementations
|
||||
{
|
||||
static std::vector<StoreFactory> * registered;
|
||||
|
||||
RegisterStoreImplementation(OpenStore fun)
|
||||
template<typename T, typename TConfig>
|
||||
static void add()
|
||||
{
|
||||
if (!implementations) implementations = new Implementations;
|
||||
implementations->push_back(fun);
|
||||
if (!registered) registered = new std::vector<StoreFactory>();
|
||||
StoreFactory factory{
|
||||
.uriSchemes = T::uriSchemes(),
|
||||
.create =
|
||||
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{ return std::make_shared<T>(scheme, uri, params); }),
|
||||
.getConfig =
|
||||
([]()
|
||||
-> std::shared_ptr<StoreConfig>
|
||||
{ return std::make_shared<TConfig>(StringMap({})); })
|
||||
};
|
||||
registered->push_back(factory);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T, typename TConfig>
|
||||
struct RegisterStoreImplementation
|
||||
{
|
||||
RegisterStoreImplementation()
|
||||
{
|
||||
Implementations::add<T, TConfig>();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/* Display a set of paths in human-readable form (i.e., between quotes
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue