nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-15 15:02:42 +01:00
Code Activity

Backport libfetchers from the flakes branch

Browse source 
This provides a pluggable mechanism for defining new fetchers. It adds
a builtin function 'fetchTree' that generalizes existing fetchers like
'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a
set of attributes, e.g.

  fetchTree {
    type = "git";
    url = "https://example.org/repo.git";
    ref = "some-branch";
    rev = "abcdef...";
  }

The existing fetchers are just wrappers around this. Note that the
input attributes to fetchTree are the same as flake input
specifications and flake lock file entries.

All fetchers share a common cache stored in
~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching
mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}).

This also adds support for Git worktrees (c169ea5904).
This commit is contained in:
Eelco Dolstra 2020-03-30 16:04:18 +02:00
parent ebb20a5356
commit 462421d345
36 changed files with 2199 additions and 647 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/libstore/derivations.cc
View file

@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (h == drvHashes.end()) {
assert(store.isValidPath(i.first));
h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
readDerivation(store, store.toRealPath(i.first)), false)).first;
}
inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
}

137
src/libstore/download.cc
View file

@ -35,10 +35,6 @@ DownloadSettings downloadSettings;
static GlobalConfig::Register r1(&downloadSettings);
CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
: uri(uri), ttl(settings.tarballTtl)
{ }
std::string resolveUri(const std::string & uri)
{
if (uri.compare(0, 8, "channel:") == 0)
@ -802,139 +798,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}
}
CachedDownloadResult Downloader::downloadCached(
ref<Store> store, const CachedDownloadRequest & request)
{
auto url = resolveUri(request.uri);
auto name = request.name;
if (name == "") {
auto p = url.rfind('/');
if (p != string::npos) name = string(url, p + 1);
}
std::optional<StorePath> expectedStorePath;
if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(*expectedStorePath)) {
CachedDownloadResult result;
result.storePath = store->printStorePath(*expectedStorePath);
result.path = store->toRealPath(result.storePath);
return result;
}
}
Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
Path dataFile = cacheDir + "/" + urlHash + ".info";
Path fileLink = cacheDir + "/" + urlHash + "-file";
PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
std::optional<StorePath> storePath;
string expectedETag;
bool skip = false;
CachedDownloadResult result;
if (pathExists(fileLink) && pathExists(dataFile)) {
storePath = store->parseStorePath(readLink(fileLink));
// FIXME
store->addTempRoot(*storePath);
if (store->isValidPath(*storePath)) {
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
skip = true;
result.effectiveUri = request.uri;
result.etag = ss[1];
} else if (!ss[1].empty()) {
debug(format("verifying previous ETag '%1%'") % ss[1]);
expectedETag = ss[1];
}
}
} else
storePath.reset();
}
if (!skip) {
try {
DownloadRequest request2(url);
request2.expectedETag = expectedETag;
auto res = download(request2);
result.effectiveUri = res.effectiveUri;
result.etag = res.etag;
if (!res.cached) {
StringSink sink;
dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = info.path.clone();
}
assert(storePath);
replaceSymlink(store->printStorePath(*storePath), fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError & e) {
if (!storePath) throw;
warn("warning: %s; using cached result", e.msg());
result.etag = expectedETag;
}
}
if (request.unpack) {
Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
std::optional<StorePath> unpackedStorePath;
if (pathExists(unpackedLink)) {
unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
// FIXME
store->addTempRoot(*unpackedStorePath);
if (!store->isValidPath(*unpackedStorePath))
unpackedStorePath.reset();
}
if (!unpackedStorePath) {
printInfo("unpacking '%s'...", url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
}
replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
storePath = std::move(*unpackedStorePath);
}
if (expectedStorePath && *storePath != *expectedStorePath) {
unsigned int statusCode = 102;
Hash gotHash = request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
: hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
url, request.expectedHash.to_string(), gotHash.to_string());
}
result.storePath = store->printStorePath(*storePath);
result.path = store->toRealPath(result.storePath);
return result;
}
bool isUri(const string & s)
{

31
src/libstore/download.hh
View file

@ -65,28 +65,6 @@ struct DownloadResult
uint64_t bodySize = 0;
};
struct CachedDownloadRequest
{
std::string uri;
bool unpack = false;
std::string name;
Hash expectedHash;
unsigned int ttl;
CachedDownloadRequest(const std::string & uri);
CachedDownloadRequest() = delete;
};
struct CachedDownloadResult
{
// Note: 'storePath' may be different from 'path' when using a
// chroot store.
Path storePath;
Path path;
std::optional<std::string> etag;
std::string effectiveUri;
};
class Store;
struct Downloader
@ -108,12 +86,6 @@ struct Downloader
invoked on the thread of the caller. */
void download(DownloadRequest && request, Sink & sink);
/* Check if the specified file is already in ~/.cache/nix/tarballs
and is more recent than ‘tarball-ttl’ seconds. Otherwise,
use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */
CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};
@ -135,4 +107,7 @@ public:
bool isUri(const string & s);
/* Resolve deprecated 'channel:<foo>' URLs. */
std::string resolveUri(const std::string & uri);
}

9
src/libstore/globals.hh
View file

@ -351,12 +351,21 @@ public:
Setting<Paths> pluginFiles{this, {}, "plugin-files",
"Plugins to dynamically load at nix initialization time."};
Setting<std::string> githubAccessToken{this, "", "github-access-token",
"GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
bool isExperimentalFeatureEnabled(const std::string & name);
void requireExperimentalFeature(const std::string & name);
Setting<bool> allowDirty{this, true, "allow-dirty",
"Whether to allow dirty Git/Mercurial trees."};
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
};

25
src/libstore/store-api.cc
View file

@ -6,6 +6,7 @@
#include "thread-pool.hh"
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
#include <future>
@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
throw NotInStore("path '%1%' is not in the Nix store", path);
return path;
}
@ -866,27 +867,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
auto e = s.find('=');
if (e != std::string::npos) {
auto value = s.substr(e + 1);
std::string decoded;
for (size_t i = 0; i < value.size(); ) {
if (value[i] == '%') {
if (i + 2 >= value.size())
throw Error("invalid URI parameter '%s'", value);
try {
decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
i += 3;
} catch (...) {
throw Error("invalid URI parameter '%s'", value);
}
} else
decoded += value[i++];
}
params[s.substr(0, e)] = decoded;
}
}
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};

1
src/libstore/store-api.hh
View file

@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
struct BasicDerivation;