nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-08 19:46:02 +01:00
Code Activity

refactor(libstore): expose HttpBinaryCacheStore and add S3BinaryCacheStore

Browse source 
Move HttpBinaryCacheStore class from .cc file to header to enable
inheritance by S3BinaryCacheStore. Create S3BinaryCacheStore class that
overrides upsertFile() to implement multipart upload logic.
This commit is contained in:
Bernardo Meurer Costa 2025-10-21 08:45:10 +00:00
parent 1a9ba0d6fe
commit 476c21d5ef
No known key found for this signature in database
4 changed files with 284 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

58
src/libstore/http-binary-cache-store.cc
View file

@ -51,23 +51,7 @@ std::string HttpBinaryCacheStoreConfig::doc()
;
}
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config)
HttpBinaryCacheStore::HttpBinaryCacheStore(ref<Config> config)
: Store{*config} // TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
@ -75,7 +59,7 @@ public:
diskCache = getNarInfoDiskCache();
}
void init() override
void HttpBinaryCacheStore::init()
{
// FIXME: do this lazily?
// For consistent cache key handling, use the reference without parameters
@ -95,9 +79,7 @@ public:
}
}
protected:
std::optional<std::string> getCompressionMethod(const std::string & path)
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
{
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
return config->narinfoCompression;
@ -109,7 +91,7 @@ protected:
return std::nullopt;
}
void maybeDisable()
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
@ -120,7 +102,7 @@ protected:
}
}
void checkEnabled()
void HttpBinaryCacheStore::checkEnabled()
{
auto state(_state.lock());
if (state->enabled)
@ -133,7 +115,7 @@ protected:
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
}
bool fileExists(const std::string & path) override
bool HttpBinaryCacheStore::fileExists(const std::string & path)
{
checkEnabled();
@ -152,17 +134,19 @@ protected:
}
}
void upsertFile(
void HttpBinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override
uint64_t sizeHint)
{
auto req = makeRequest(path);
auto data = StreamToSourceAdapter(istream).drain();
if (auto compressionMethod = getCompressionMethod(path)) {
auto compressionMethod = getCompressionMethod(path);
if (compressionMethod) {
data = compress(*compressionMethod, data);
req.headers.emplace_back("Content-Encoding", *compressionMethod);
}
@ -173,12 +157,11 @@ protected:
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP(
"while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
}
}
FileTransferRequest makeRequest(const std::string & path)
FileTransferRequest HttpBinaryCacheStore::makeRequest(const std::string & path)
{
/* Otherwise the last path fragment will get discarded. */
auto cacheUriWithTrailingSlash = config->cacheUri;
@ -202,7 +185,7 @@ protected:
return FileTransferRequest(result);
}
void getFile(const std::string & path, Sink & sink) override
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
checkEnabled();
auto request(makeRequest(path));
@ -217,7 +200,7 @@ protected:
}
}
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override
void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
@ -226,12 +209,12 @@ protected:
auto request(makeRequest(path));
getFileTransfer()->enqueueFileTransfer(
request, {[callbackPtr, this](std::future<FileTransferResult> result) {
getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
if (e.error == FileTransfer::NotFound
|| e.error == FileTransfer::Forbidden)
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
@ -246,7 +229,7 @@ protected:
}
}
std::optional<std::string> getNixCacheInfo() override
std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
{
try {
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
@ -267,11 +250,10 @@ protected:
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
std::optional<TrustedFlag> HttpBinaryCacheStore::isTrustedClient()
{
return std::nullopt;
}
};
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{

51
src/libstore/include/nix/store/http-binary-cache-store.hh
View file

@ -3,6 +3,10 @@
#include "nix/util/url.hh"
#include "nix/store/binary-cache-store.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/sync.hh"
#include <chrono>
namespace nix {
@ -46,4 +50,51 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
StoreReference getReference() const override;
};
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config);
void init() override;
protected:
std::optional<std::string> getCompressionMethod(const std::string & path);
void maybeDisable();
void checkEnabled();
bool fileExists(const std::string & path) override;
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
FileTransferRequest makeRequest(const std::string & path);
void getFile(const std::string & path, Sink & sink) override;
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override;
std::optional<std::string> getNixCacheInfo() override;
std::optional<TrustedFlag> isTrustedClient() override;
};
} // namespace nix

2
src/libstore/include/nix/store/s3-binary-cache-store.hh
View file

@ -77,6 +77,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
static std::string doc();
std::string getHumanReadableURI() const override;
ref<Store> openStore() const override;
};
} // namespace nix

37
src/libstore/s3-binary-cache-store.cc
View file

@ -7,6 +7,36 @@
namespace nix {
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
{
public:
S3BinaryCacheStore(ref<S3BinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore{config}
, s3Config{config}
{
}
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
private:
ref<S3BinaryCacheStoreConfig> s3Config;
};
void S3BinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint)
{
HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
}
StringSet S3BinaryCacheStoreConfig::uriSchemes()
{
return {"s3"};
@ -51,6 +81,13 @@ std::string S3BinaryCacheStoreConfig::doc()
)";
}
ref<Store> S3BinaryCacheStoreConfig::openStore() const
{
auto sharedThis = std::const_pointer_cast<S3BinaryCacheStoreConfig>(
std::static_pointer_cast<const S3BinaryCacheStoreConfig>(shared_from_this()));
return make_ref<S3BinaryCacheStore>(ref{sharedThis});
}
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
} // namespace nix