#include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" #include "nix/store/globals.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/callback.hh" #include "nix/store/store-registration.hh" #include "nix/util/compression.hh" namespace nix { MakeError(UploadToHTTP, Error); StringSet HttpBinaryCacheStoreConfig::uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; auto ret = StringSet{"http", "https"}; if (forceHttp) ret.insert("file"); return ret; } HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , cacheUri(parseURL( std::string{scheme} + "://" + (!_cacheUri.empty() ? _cacheUri : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme)))) { while (!cacheUri.path.empty() && cacheUri.path.back() == "") cacheUri.path.pop_back(); } StoreReference HttpBinaryCacheStoreConfig::getReference() const { return { .variant = StoreReference::Specified{ .scheme = cacheUri.scheme, .authority = cacheUri.renderAuthorityAndPath(), }, .params = getQueryParams(), }; } std::string HttpBinaryCacheStoreConfig::doc() { return #include "http-binary-cache-store.md" ; } class HttpBinaryCacheStore : public virtual BinaryCacheStore { struct State { bool enabled = true; std::chrono::steady_clock::time_point disabledUntil; }; Sync _state; public: using Config = HttpBinaryCacheStoreConfig; ref config; HttpBinaryCacheStore(ref config) : Store{*config} // TODO it will actually mutate the configuration , BinaryCacheStore{*config} , config{config} { diskCache = getNarInfoDiskCache(); } void init() override { // FIXME: do this lazily? // For consistent cache key handling, use the reference without parameters // This matches what's used in Store::queryPathInfo() lookups auto cacheKey = config->getReference().render(/*withParams=*/false); if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); config->priority.setDefault(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); } } protected: void maybeDisable() { auto state(_state.lock()); if (state->enabled && settings.tryFallback) { int t = 60; printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t); state->enabled = false; state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); } } void checkEnabled() { auto state(_state.lock()); if (state->enabled) return; if (std::chrono::steady_clock::now() > state->disabledUntil) { state->enabled = true; debug("re-enabling binary cache '%s'", config->getHumanReadableURI()); return; } throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI()); } bool fileExists(const std::string & path) override { checkEnabled(); try { FileTransferRequest request(makeRequest(path)); request.head = true; getFileTransfer()->download(request); return true; } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; } } void upsertFile( const std::string & path, std::shared_ptr> istream, const std::string & mimeType) override { auto req = makeRequest(path); auto data = StreamToSourceAdapter(istream).drain(); // Determine compression method based on file type std::string compressionMethod; if (hasSuffix(path, ".narinfo")) compressionMethod = config->narinfoCompression; else if (hasSuffix(path, ".ls")) compressionMethod = config->lsCompression; else if (hasPrefix(path, "log/")) compressionMethod = config->logCompression; // Apply compression if configured if (!compressionMethod.empty()) { data = compress(compressionMethod, data); req.headers.emplace_back("Content-Encoding", compressionMethod); } req.data = std::move(data); req.mimeType = mimeType; try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { throw UploadToHTTP( "while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg()); } } FileTransferRequest makeRequest(const std::string & path) { /* Otherwise the last path fragment will get discarded. */ auto cacheUriWithTrailingSlash = config->cacheUri; if (!cacheUriWithTrailingSlash.path.empty()) cacheUriWithTrailingSlash.path.push_back(""); /* path is not a path, but a full relative or absolute URL, e.g. we've seen in the wild NARINFO files have a URL field which is `nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj` (note the query param) and that gets passed here. */ auto result = parseURLRelative(path, cacheUriWithTrailingSlash); /* For S3 URLs, preserve query parameters from the base URL when the relative path doesn't have its own query parameters. This is needed to preserve S3-specific parameters like endpoint and region. */ if (config->cacheUri.scheme == "s3" && result.query.empty()) { result.query = config->cacheUri.query; } return FileTransferRequest(result); } void getFile(const std::string & path, Sink & sink) override { checkEnabled(); auto request(makeRequest(path)); try { getFileTransfer()->download(std::move(request), sink); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile( "file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI()); maybeDisable(); throw; } } void getFile(const std::string & path, Callback> callback) noexcept override { auto callbackPtr = std::make_shared(std::move(callback)); try { checkEnabled(); auto request(makeRequest(path)); getFileTransfer()->enqueueFileTransfer( request, {[callbackPtr, this](std::future result) { try { (*callbackPtr)(std::move(result.get().data)); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)({}); maybeDisable(); callbackPtr->rethrow(); } catch (...) { callbackPtr->rethrow(); } }}); } catch (...) { callbackPtr->rethrow(); return; } } std::optional getNixCacheInfo() override { try { auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); return result.data; } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound) return std::nullopt; maybeDisable(); throw; } } /** * This isn't actually necessary read only. We support "upsert" now, so we * have a notion of authentication via HTTP POST/PUT. * * For now, we conservatively say we don't know. * * \todo try to expose our HTTP authentication status. */ std::optional isTrustedClient() override { return std::nullopt; } }; ref HttpBinaryCacheStore::Config::openStore() const { return make_ref( ref{// FIXME we shouldn't actually need a mutable config std::const_pointer_cast(shared_from_this())}); } static RegisterStoreImplementation regHttpBinaryCacheStore; } // namespace nix