mirror of
https://github.com/NixOS/nix.git
synced 2025-11-21 01:39:36 +01:00
Get rid of downloadCached()
Everything uses the generic caching system now.
This commit is contained in:
Eelco Dolstra
parent
c5ec95e2c7
commit
f6ddf48882
9 changed files with 113 additions and 260 deletions
|
|
@ -35,10 +35,6 @@ DownloadSettings downloadSettings;
|
|||
|
||||
static GlobalConfig::Register r1(&downloadSettings);
|
||||
|
||||
CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
|
||||
: uri(uri), ttl(settings.tarballTtl)
|
||||
{ }
|
||||
|
||||
std::string resolveUri(const std::string & uri)
|
||||
{
|
||||
if (uri.compare(0, 8, "channel:") == 0)
|
||||
|
|
@ -801,148 +797,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
|
|||
}
|
||||
}
|
||||
|
||||
CachedDownloadResult Downloader::downloadCached(
|
||||
ref<Store> store, const CachedDownloadRequest & request)
|
||||
{
|
||||
auto url = resolveUri(request.uri);
|
||||
|
||||
auto name = request.name;
|
||||
if (name == "") {
|
||||
auto p = url.rfind('/');
|
||||
if (p != string::npos) name = string(url, p + 1);
|
||||
}
|
||||
|
||||
std::optional<StorePath> expectedStorePath;
|
||||
if (request.expectedHash) {
|
||||
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
|
||||
if (store->isValidPath(*expectedStorePath)) {
|
||||
CachedDownloadResult result;
|
||||
result.storePath = store->printStorePath(*expectedStorePath);
|
||||
result.path = store->toRealPath(result.storePath);
|
||||
assert(!request.getLastModified); // FIXME
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
Path cacheDir = getCacheDir() + "/nix/tarballs";
|
||||
createDirs(cacheDir);
|
||||
|
||||
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
|
||||
|
||||
Path dataFile = cacheDir + "/" + urlHash + ".info";
|
||||
Path fileLink = cacheDir + "/" + urlHash + "-file";
|
||||
|
||||
PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
|
||||
|
||||
std::optional<StorePath> storePath;
|
||||
|
||||
string expectedETag;
|
||||
|
||||
bool skip = false;
|
||||
|
||||
CachedDownloadResult result;
|
||||
|
||||
if (pathExists(fileLink) && pathExists(dataFile)) {
|
||||
storePath = store->parseStorePath(readLink(fileLink));
|
||||
// FIXME
|
||||
store->addTempRoot(*storePath);
|
||||
if (store->isValidPath(*storePath)) {
|
||||
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
|
||||
if (ss.size() >= 3 && ss[0] == url) {
|
||||
time_t lastChecked;
|
||||
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
|
||||
skip = true;
|
||||
result.effectiveUri = request.uri;
|
||||
result.etag = ss[1];
|
||||
} else if (!ss[1].empty()) {
|
||||
debug(format("verifying previous ETag '%1%'") % ss[1]);
|
||||
expectedETag = ss[1];
|
||||
}
|
||||
}
|
||||
} else
|
||||
storePath.reset();
|
||||
}
|
||||
|
||||
if (!skip) {
|
||||
|
||||
try {
|
||||
DownloadRequest request2(url);
|
||||
request2.expectedETag = expectedETag;
|
||||
auto res = download(request2);
|
||||
result.effectiveUri = res.effectiveUri;
|
||||
result.etag = res.etag;
|
||||
|
||||
if (!res.cached) {
|
||||
StringSink sink;
|
||||
dumpString(*res.data, sink);
|
||||
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
|
||||
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
|
||||
info.narHash = hashString(htSHA256, *sink.s);
|
||||
info.narSize = sink.s->size();
|
||||
info.ca = makeFixedOutputCA(false, hash);
|
||||
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
|
||||
storePath = info.path.clone();
|
||||
}
|
||||
|
||||
assert(storePath);
|
||||
replaceSymlink(store->printStorePath(*storePath), fileLink);
|
||||
|
||||
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
|
||||
} catch (DownloadError & e) {
|
||||
if (!storePath) throw;
|
||||
warn("warning: %s; using cached result", e.msg());
|
||||
result.etag = expectedETag;
|
||||
}
|
||||
}
|
||||
|
||||
if (request.unpack) {
|
||||
Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
|
||||
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
|
||||
std::optional<StorePath> unpackedStorePath;
|
||||
if (pathExists(unpackedLink)) {
|
||||
unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
|
||||
store->addTempRoot(*unpackedStorePath);
|
||||
if (!store->isValidPath(*unpackedStorePath))
|
||||
unpackedStorePath.reset();
|
||||
else
|
||||
result.lastModified = lstat(unpackedLink).st_mtime;
|
||||
}
|
||||
if (!unpackedStorePath) {
|
||||
printInfo("unpacking '%s'...", url);
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete autoDelete(tmpDir, true);
|
||||
unpackTarfile(store->toRealPath(*storePath), tmpDir);
|
||||
auto members = readDirectory(tmpDir);
|
||||
if (members.size() != 1)
|
||||
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
|
||||
auto topDir = tmpDir + "/" + members.begin()->name;
|
||||
result.lastModified = lstat(topDir).st_mtime;
|
||||
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
|
||||
}
|
||||
// Store the last-modified date of the tarball in the symlink
|
||||
// mtime. This saves us from having to store it somewhere
|
||||
// else.
|
||||
replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified);
|
||||
storePath = std::move(*unpackedStorePath);
|
||||
}
|
||||
|
||||
if (expectedStorePath && *storePath != *expectedStorePath) {
|
||||
unsigned int statusCode = 102;
|
||||
Hash gotHash = request.unpack
|
||||
? hashPath(request.expectedHash.type, store->toRealPath(*storePath)).first
|
||||
: hashFile(request.expectedHash.type, store->toRealPath(*storePath));
|
||||
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
|
||||
url, request.expectedHash.to_string(), gotHash.to_string());
|
||||
}
|
||||
|
||||
if (request.gcRoot)
|
||||
store->addIndirectRoot(fileLink);
|
||||
|
||||
result.storePath = store->printStorePath(*storePath);
|
||||
result.path = store->toRealPath(result.storePath);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
bool isUri(const string & s)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -65,31 +65,6 @@ struct DownloadResult
|
|||
uint64_t bodySize = 0;
|
||||
};
|
||||
|
||||
struct CachedDownloadRequest
|
||||
{
|
||||
std::string uri;
|
||||
bool unpack = false;
|
||||
std::string name;
|
||||
Hash expectedHash;
|
||||
unsigned int ttl;
|
||||
bool gcRoot = false;
|
||||
bool getLastModified = false;
|
||||
|
||||
CachedDownloadRequest(const std::string & uri);
|
||||
CachedDownloadRequest() = delete;
|
||||
};
|
||||
|
||||
struct CachedDownloadResult
|
||||
{
|
||||
// Note: 'storePath' may be different from 'path' when using a
|
||||
// chroot store.
|
||||
Path storePath;
|
||||
Path path;
|
||||
std::optional<std::string> etag;
|
||||
std::string effectiveUri;
|
||||
std::optional<time_t> lastModified;
|
||||
};
|
||||
|
||||
class Store;
|
||||
|
||||
struct Downloader
|
||||
|
|
@ -111,12 +86,6 @@ struct Downloader
|
|||
invoked on the thread of the caller. */
|
||||
void download(DownloadRequest && request, Sink & sink);
|
||||
|
||||
/* Check if the specified file is already in ~/.cache/nix/tarballs
|
||||
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
|
||||
use the recorded ETag to verify if the server has a more
|
||||
recent version, and if so, download it to the Nix store. */
|
||||
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
|
||||
|
||||
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
|
||||
};
|
||||
|
||||
|
|
@ -138,4 +107,7 @@ public:
|
|||
|
||||
bool isUri(const string & s);
|
||||
|
||||
/* Resolve deprecated 'channel:<foo>' URLs. */
|
||||
std::string resolveUri(const std::string & uri);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ struct DownloadFileResult
|
|||
{
|
||||
StorePath storePath;
|
||||
std::string etag;
|
||||
std::string effectiveUrl;
|
||||
};
|
||||
|
||||
DownloadFileResult downloadFile(
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ DownloadFileResult downloadFile(
|
|||
if (cached && !cached->expired)
|
||||
return {
|
||||
.storePath = std::move(cached->storePath),
|
||||
.etag = getStrAttr(cached->infoAttrs, "etag")
|
||||
.etag = getStrAttr(cached->infoAttrs, "etag"),
|
||||
.effectiveUrl = getStrAttr(cached->infoAttrs, "url")
|
||||
};
|
||||
|
||||
DownloadRequest request(url);
|
||||
|
|
@ -40,6 +41,7 @@ DownloadFileResult downloadFile(
|
|||
|
||||
Attrs infoAttrs({
|
||||
{"etag", res.etag},
|
||||
{"url", res.effectiveUri},
|
||||
});
|
||||
|
||||
std::optional<StorePath> storePath;
|
||||
|
|
@ -67,9 +69,22 @@ DownloadFileResult downloadFile(
|
|||
*storePath,
|
||||
immutable);
|
||||
|
||||
if (url != res.effectiveUri)
|
||||
getCache()->add(
|
||||
store,
|
||||
{
|
||||
{"type", "file"},
|
||||
{"url", res.effectiveUri},
|
||||
{"name", name},
|
||||
},
|
||||
infoAttrs,
|
||||
*storePath,
|
||||
immutable);
|
||||
|
||||
return {
|
||||
.storePath = std::move(*storePath),
|
||||
.etag = res.etag,
|
||||
.effectiveUrl = res.effectiveUri,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue