nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 03:56:01 +01:00
Code Activity

refactor(libstore): expose HttpBinaryCacheStore and add S3BinaryCacheStore

Browse source 
Move HttpBinaryCacheStore class from .cc file to header to enable
inheritance by S3BinaryCacheStore. Create S3BinaryCacheStore class that
overrides upsertFile() to implement multipart upload logic.
This commit is contained in:
Bernardo Meurer Costa 2025-10-21 08:45:10 +00:00
parent 1a9ba0d6fe
commit 476c21d5ef
No known key found for this signature in database
4 changed files with 284 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

108
src/libstore/http-binary-cache-store.cc
View file

@ -51,32 +51,16 @@ std::string HttpBinaryCacheStoreConfig::doc()
;
}
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config)
HttpBinaryCacheStore::HttpBinaryCacheStore(ref<Config> config)
: Store{*config} // TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
{
{
diskCache = getNarInfoDiskCache();
}
}
void init() override
{
void HttpBinaryCacheStore::init()
{
// FIXME: do this lazily?
// For consistent cache key handling, use the reference without parameters
// This matches what's used in Store::queryPathInfo() lookups
@ -93,12 +77,10 @@ public:
}
diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
}
}
}
protected:
std::optional<std::string> getCompressionMethod(const std::string & path)
{
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
{
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
return config->narinfoCompression;
else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty())
@ -107,10 +89,10 @@ protected:
return config->logCompression;
else
return std::nullopt;
}
}
void maybeDisable()
{
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
@ -118,10 +100,10 @@ protected:
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
}
void checkEnabled()
{
void HttpBinaryCacheStore::checkEnabled()
{
auto state(_state.lock());
if (state->enabled)
return;
@ -131,10 +113,10 @@ protected:
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
}
}
bool fileExists(const std::string & path) override
{
bool HttpBinaryCacheStore::fileExists(const std::string & path)
{
checkEnabled();
try {
@ -150,19 +132,21 @@ protected:
maybeDisable();
throw;
}
}
}
void upsertFile(
void HttpBinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override
{
uint64_t sizeHint)
{
auto req = makeRequest(path);
auto data = StreamToSourceAdapter(istream).drain();
if (auto compressionMethod = getCompressionMethod(path)) {
auto compressionMethod = getCompressionMethod(path);
if (compressionMethod) {
data = compress(*compressionMethod, data);
req.headers.emplace_back("Content-Encoding", *compressionMethod);
}
@ -173,13 +157,12 @@ protected:
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP(
"while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
}
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
}
}
FileTransferRequest makeRequest(const std::string & path)
{
FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path)
{
/* Otherwise the last path fragment will get discarded. */
auto cacheUriWithTrailingSlash = config->cacheUri;
if (!cacheUriWithTrailingSlash.path.empty())
@ -200,10 +183,10 @@ protected:
}
return FileTransferRequest(result);
}
}
void getFile(const std::string & path, Sink & sink) override
{
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
checkEnabled();
auto request(makeRequest(path));
try {
@ -215,10 +198,10 @@ protected:
maybeDisable();
throw;
}
}
}
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override
{
void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
@ -226,12 +209,12 @@ protected:
auto request(makeRequest(path));
getFileTransfer()->enqueueFileTransfer(
request, {[callbackPtr, this](std::future<FileTransferResult> result) {
getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
if (e.error == FileTransfer::NotFound
|| e.error == FileTransfer::Forbidden)
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
@ -244,10 +227,10 @@ protected:
callbackPtr->rethrow();
return;
}
}
}
std::optional<std::string> getNixCacheInfo() override
{
std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
{
try {
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
return result.data;
@ -257,9 +240,9 @@ protected:
maybeDisable();
throw;
}
}
}
/**
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
@ -267,11 +250,10 @@ protected:
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
std::optional<TrustedFlag> HttpBinaryCacheStore::isTrustedClient()
{
return std::nullopt;
}
};
}
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{

51
src/libstore/include/nix/store/http-binary-cache-store.hh
View file

@ -3,6 +3,10 @@
#include "nix/util/url.hh"
#include "nix/store/binary-cache-store.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/sync.hh"
#include <chrono>
namespace nix {
@ -46,4 +50,51 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
StoreReference getReference() const override;
};
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config);
void init() override;
protected:
std::optional<std::string> getCompressionMethod(const std::string & path);
void maybeDisable();
void checkEnabled();
bool fileExists(const std::string & path) override;
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
FileTransferRequest makeRequest(const std::string & path);
void getFile(const std::string & path, Sink & sink) override;
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override;
std::optional<std::string> getNixCacheInfo() override;
std::optional<TrustedFlag> isTrustedClient() override;
};
} // namespace nix

2
src/libstore/include/nix/store/s3-binary-cache-store.hh
View file

@ -77,6 +77,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
static std::string doc();
std::string getHumanReadableURI() const override;
ref<Store> openStore() const override;
};
} // namespace nix

37
src/libstore/s3-binary-cache-store.cc
View file

@ -7,6 +7,36 @@
namespace nix {
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
{
public:
S3BinaryCacheStore(ref<S3BinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore{config}
, s3Config{config}
{
}
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
private:
ref<S3BinaryCacheStoreConfig> s3Config;
};
void S3BinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint)
{
HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
}
StringSet S3BinaryCacheStoreConfig::uriSchemes()
{
return {"s3"};
@ -51,6 +81,13 @@ std::string S3BinaryCacheStoreConfig::doc()
)";
}
ref<Store> S3BinaryCacheStoreConfig::openStore() const
{
auto sharedThis = std::const_pointer_cast<S3BinaryCacheStoreConfig>(
std::static_pointer_cast<const S3BinaryCacheStoreConfig>(shared_from_this()));
return make_ref<S3BinaryCacheStore>(ref{sharedThis});
}
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
} // namespace nix