nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00
Code Activity

Merge pull request #14350 from lovesegfault/s3-binary-cache-store

Browse source 
refactor(libstore): expose HttpBinaryCacheStore and add S3BinaryCacheStore
This commit is contained in:
Sergei Zimmerman 2025-10-24 22:59:02 +00:00 • committed by GitHub
commit 8b7e03f0f9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 284 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

58
src/libstore/http-binary-cache-store.cc
View file

@ -51,23 +51,7 @@ std::string HttpBinaryCacheStoreConfig::doc()
;
}
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config)
HttpBinaryCacheStore::HttpBinaryCacheStore(ref<Config> config)
: Store{*config} // TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
@ -75,7 +59,7 @@ public:
diskCache = getNarInfoDiskCache();
}
void init() override
void HttpBinaryCacheStore::init()
{
// FIXME: do this lazily?
// For consistent cache key handling, use the reference without parameters
@ -95,9 +79,7 @@ public:
}
}
protected:
std::optional<std::string> getCompressionMethod(const std::string & path)
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
{
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
return config->narinfoCompression;
@ -109,7 +91,7 @@ protected:
return std::nullopt;
}
void maybeDisable()
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
@ -120,7 +102,7 @@ protected:
}
}
void checkEnabled()
void HttpBinaryCacheStore::checkEnabled()
{
auto state(_state.lock());
if (state->enabled)
@ -133,7 +115,7 @@ protected:
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
}
bool fileExists(const std::string & path) override
bool HttpBinaryCacheStore::fileExists(const std::string & path)
{
checkEnabled();
@ -152,17 +134,19 @@ protected:
}
}
void upsertFile(
void HttpBinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override
uint64_t sizeHint)
{
auto req = makeRequest(path);
auto data = StreamToSourceAdapter(istream).drain();
if (auto compressionMethod = getCompressionMethod(path)) {
auto compressionMethod = getCompressionMethod(path);
if (compressionMethod) {
data = compress(*compressionMethod, data);
req.headers.emplace_back("Content-Encoding", *compressionMethod);
}
@ -173,12 +157,11 @@ protected:
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP(
"while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
}
}
FileTransferRequest makeRequest(const std::string & path)
FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path)
{
/* Otherwise the last path fragment will get discarded. */
auto cacheUriWithTrailingSlash = config->cacheUri;
@ -202,7 +185,7 @@ protected:
return FileTransferRequest(result);
}
void getFile(const std::string & path, Sink & sink) override
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
checkEnabled();
auto request(makeRequest(path));
@ -217,7 +200,7 @@ protected:
}
}
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override
void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
@ -226,12 +209,12 @@ protected:
auto request(makeRequest(path));
getFileTransfer()->enqueueFileTransfer(
request, {[callbackPtr, this](std::future<FileTransferResult> result) {
getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
if (e.error == FileTransfer::NotFound
|| e.error == FileTransfer::Forbidden)
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
@ -246,7 +229,7 @@ protected:
}
}
std::optional<std::string> getNixCacheInfo() override
std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
{
try {
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
@ -267,11 +250,10 @@ protected:
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
std::optional<TrustedFlag> HttpBinaryCacheStore::isTrustedClient()
{
return std::nullopt;
}
};
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{

51
src/libstore/include/nix/store/http-binary-cache-store.hh
View file

@ -3,6 +3,10 @@
#include "nix/util/url.hh"
#include "nix/store/binary-cache-store.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/sync.hh"
#include <chrono>
namespace nix {
@ -46,4 +50,51 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
StoreReference getReference() const override;
};
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config);
void init() override;
protected:
std::optional<std::string> getCompressionMethod(const std::string & path);
void maybeDisable();
void checkEnabled();
bool fileExists(const std::string & path) override;
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
FileTransferRequest makeRequest(const std::string & path);
void getFile(const std::string & path, Sink & sink) override;
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override;
std::optional<std::string> getNixCacheInfo() override;
std::optional<TrustedFlag> isTrustedClient() override;
};
} // namespace nix

2
src/libstore/include/nix/store/s3-binary-cache-store.hh
View file

@ -77,6 +77,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
static std::string doc();
std::string getHumanReadableURI() const override;
ref<Store> openStore() const override;
};
} // namespace nix

37
src/libstore/s3-binary-cache-store.cc
View file

@ -7,6 +7,36 @@
namespace nix {
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
{
public:
S3BinaryCacheStore(ref<S3BinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore{config}
, s3Config{config}
{
}
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
private:
ref<S3BinaryCacheStoreConfig> s3Config;
};
void S3BinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint)
{
HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
}
StringSet S3BinaryCacheStoreConfig::uriSchemes()
{
return {"s3"};
@ -51,6 +81,13 @@ std::string S3BinaryCacheStoreConfig::doc()
)";
}
ref<Store> S3BinaryCacheStoreConfig::openStore() const
{
auto sharedThis = std::const_pointer_cast<S3BinaryCacheStoreConfig>(
std::static_pointer_cast<const S3BinaryCacheStoreConfig>(shared_from_this()));
return make_ref<S3BinaryCacheStore>(ref{sharedThis});
}
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
} // namespace nix