mirror of
https://github.com/NixOS/nix.git
synced 2025-11-15 15:02:42 +01:00
Move fetchers from libstore to libfetchers
This commit is contained in:
Eelco Dolstra
parent
4ba4c7ff66
commit
e0a0ae0467
35 changed files with 80 additions and 90 deletions
|
|
@ -1,92 +0,0 @@
|
|||
#include "attrs.hh"
|
||||
#include "fetchers.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
Attrs jsonToAttrs(const nlohmann::json & json)
|
||||
{
|
||||
Attrs attrs;
|
||||
|
||||
for (auto & i : json.items()) {
|
||||
if (i.value().is_number())
|
||||
attrs.emplace(i.key(), i.value().get<int64_t>());
|
||||
else if (i.value().is_string())
|
||||
attrs.emplace(i.key(), i.value().get<std::string>());
|
||||
else if (i.value().is_boolean())
|
||||
attrs.emplace(i.key(), i.value().get<bool>());
|
||||
else
|
||||
throw Error("unsupported input attribute type in lock file");
|
||||
}
|
||||
|
||||
return attrs;
|
||||
}
|
||||
|
||||
nlohmann::json attrsToJson(const Attrs & attrs)
|
||||
{
|
||||
nlohmann::json json;
|
||||
for (auto & attr : attrs) {
|
||||
if (auto v = std::get_if<int64_t>(&attr.second)) {
|
||||
json[attr.first] = *v;
|
||||
} else if (auto v = std::get_if<std::string>(&attr.second)) {
|
||||
json[attr.first] = *v;
|
||||
} else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
|
||||
json[attr.first] = v->t;
|
||||
} else abort();
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto i = attrs.find(name);
|
||||
if (i == attrs.end()) return {};
|
||||
if (auto v = std::get_if<std::string>(&i->second))
|
||||
return *v;
|
||||
throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
|
||||
}
|
||||
|
||||
std::string getStrAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto s = maybeGetStrAttr(attrs, name);
|
||||
if (!s)
|
||||
throw Error("input attribute '%s' is missing", name);
|
||||
return *s;
|
||||
}
|
||||
|
||||
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto i = attrs.find(name);
|
||||
if (i == attrs.end()) return {};
|
||||
if (auto v = std::get_if<int64_t>(&i->second))
|
||||
return *v;
|
||||
throw Error("input attribute '%s' is not an integer", name);
|
||||
}
|
||||
|
||||
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto s = maybeGetIntAttr(attrs, name);
|
||||
if (!s)
|
||||
throw Error("input attribute '%s' is missing", name);
|
||||
return *s;
|
||||
}
|
||||
|
||||
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto i = attrs.find(name);
|
||||
if (i == attrs.end()) return {};
|
||||
if (auto v = std::get_if<int64_t>(&i->second))
|
||||
return *v;
|
||||
throw Error("input attribute '%s' is not a Boolean", name);
|
||||
}
|
||||
|
||||
bool getBoolAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto s = maybeGetBoolAttr(attrs, name);
|
||||
if (!s)
|
||||
throw Error("input attribute '%s' is missing", name);
|
||||
return *s;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
|
||||
#include <variant>
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
/* Wrap bools to prevent string literals (i.e. 'char *') from being
|
||||
cast to a bool in Attr. */
|
||||
template<typename T>
|
||||
struct Explicit {
|
||||
T t;
|
||||
};
|
||||
|
||||
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
|
||||
typedef std::map<std::string, Attr> Attrs;
|
||||
|
||||
Attrs jsonToAttrs(const nlohmann::json & json);
|
||||
|
||||
nlohmann::json attrsToJson(const Attrs & attrs);
|
||||
|
||||
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
std::string getStrAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
bool getBoolAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
}
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
#include "fetchers/cache.hh"
|
||||
#include "sqlite.hh"
|
||||
#include "sync.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
static const char * schema = R"sql(
|
||||
|
||||
create table if not exists Cache (
|
||||
input text not null,
|
||||
info text not null,
|
||||
path text not null,
|
||||
immutable integer not null,
|
||||
timestamp integer not null,
|
||||
primary key (input)
|
||||
);
|
||||
)sql";
|
||||
|
||||
struct CacheImpl : Cache
|
||||
{
|
||||
struct State
|
||||
{
|
||||
SQLite db;
|
||||
SQLiteStmt add, lookup;
|
||||
};
|
||||
|
||||
Sync<State> _state;
|
||||
|
||||
CacheImpl()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
|
||||
createDirs(dirOf(dbPath));
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
state->add.create(state->db,
|
||||
"insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
|
||||
|
||||
state->lookup.create(state->db,
|
||||
"select info, path, immutable, timestamp from Cache where input = ?");
|
||||
}
|
||||
|
||||
void add(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs,
|
||||
const Attrs & infoAttrs,
|
||||
const StorePath & storePath,
|
||||
bool immutable) override
|
||||
{
|
||||
_state.lock()->add.use()
|
||||
(attrsToJson(inAttrs).dump())
|
||||
(attrsToJson(infoAttrs).dump())
|
||||
(store->printStorePath(storePath))
|
||||
(immutable)
|
||||
(time(0)).exec();
|
||||
}
|
||||
|
||||
std::optional<std::pair<Attrs, StorePath>> lookup(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs) override
|
||||
{
|
||||
if (auto res = lookupExpired(store, inAttrs)) {
|
||||
if (!res->expired)
|
||||
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
|
||||
debug("ignoring expired cache entry '%s'",
|
||||
attrsToJson(inAttrs).dump());
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<Result> lookupExpired(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs) override
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
auto inAttrsJson = attrsToJson(inAttrs).dump();
|
||||
|
||||
auto stmt(state->lookup.use()(inAttrsJson));
|
||||
if (!stmt.next()) {
|
||||
debug("did not find cache entry for '%s'", inAttrsJson);
|
||||
return {};
|
||||
}
|
||||
|
||||
auto infoJson = stmt.getStr(0);
|
||||
auto storePath = store->parseStorePath(stmt.getStr(1));
|
||||
auto immutable = stmt.getInt(2) != 0;
|
||||
auto timestamp = stmt.getInt(3);
|
||||
|
||||
store->addTempRoot(storePath);
|
||||
if (!store->isValidPath(storePath)) {
|
||||
// FIXME: we could try to substitute 'storePath'.
|
||||
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
|
||||
return {};
|
||||
}
|
||||
|
||||
debug("using cache entry '%s' -> '%s', '%s'",
|
||||
inAttrsJson, infoJson, store->printStorePath(storePath));
|
||||
|
||||
return Result {
|
||||
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
|
||||
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
|
||||
.storePath = std::move(storePath)
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
ref<Cache> getCache()
|
||||
{
|
||||
static auto cache = std::make_shared<CacheImpl>();
|
||||
return ref<Cache>(cache);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "fetchers/fetchers.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Cache
|
||||
{
|
||||
virtual void add(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs,
|
||||
const Attrs & infoAttrs,
|
||||
const StorePath & storePath,
|
||||
bool immutable) = 0;
|
||||
|
||||
virtual std::optional<std::pair<Attrs, StorePath>> lookup(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs) = 0;
|
||||
|
||||
struct Result
|
||||
{
|
||||
bool expired = false;
|
||||
Attrs infoAttrs;
|
||||
StorePath storePath;
|
||||
};
|
||||
|
||||
virtual std::optional<Result> lookupExpired(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs) = 0;
|
||||
};
|
||||
|
||||
ref<Cache> getCache();
|
||||
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
#include "fetchers.hh"
|
||||
#include "parse.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
|
||||
|
||||
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
|
||||
{
|
||||
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
|
||||
inputSchemes->push_back(std::move(inputScheme));
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
|
||||
{
|
||||
for (auto & inputScheme : *inputSchemes) {
|
||||
auto res = inputScheme->inputFromURL(url);
|
||||
if (res) return res;
|
||||
}
|
||||
throw Error("input '%s' is unsupported", url.url);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const std::string & url)
|
||||
{
|
||||
return inputFromURL(parseURL(url));
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
|
||||
{
|
||||
for (auto & inputScheme : *inputSchemes) {
|
||||
auto res = inputScheme->inputFromAttrs(attrs);
|
||||
if (res) {
|
||||
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
|
||||
// FIXME: require SRI hash.
|
||||
res->narHash = Hash(*narHash);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
throw Error("input '%s' is unsupported", attrsToJson(attrs));
|
||||
}
|
||||
|
||||
Attrs Input::toAttrs() const
|
||||
{
|
||||
auto attrs = toAttrsInternal();
|
||||
if (narHash)
|
||||
attrs.emplace("narHash", narHash->to_string(SRI));
|
||||
attrs.emplace("type", type());
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
|
||||
{
|
||||
auto [tree, input] = fetchTreeInternal(store);
|
||||
|
||||
if (tree.actualPath == "")
|
||||
tree.actualPath = store->toRealPath(tree.storePath);
|
||||
|
||||
if (!tree.info.narHash)
|
||||
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
|
||||
|
||||
if (input->narHash)
|
||||
assert(input->narHash == tree.info.narHash);
|
||||
|
||||
if (narHash && narHash != input->narHash)
|
||||
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
|
||||
to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI));
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
|
||||
std::shared_ptr<const Input> Input::applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const
|
||||
{
|
||||
if (ref)
|
||||
throw Error("don't know how to apply '%s' to '%s'", *ref, to_string());
|
||||
if (rev)
|
||||
throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string());
|
||||
return shared_from_this();
|
||||
}
|
||||
|
||||
StorePath TreeInfo::computeStorePath(Store & store) const
|
||||
{
|
||||
assert(narHash);
|
||||
return store.makeFixedOutputPath(true, narHash, "source");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "hash.hh"
|
||||
#include "path.hh"
|
||||
#include "tree-info.hh"
|
||||
#include "attrs.hh"
|
||||
#include "parse.hh"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Input;
|
||||
|
||||
struct Tree
|
||||
{
|
||||
Path actualPath;
|
||||
StorePath storePath;
|
||||
TreeInfo info;
|
||||
};
|
||||
|
||||
struct Input : std::enable_shared_from_this<Input>
|
||||
{
|
||||
std::optional<Hash> narHash; // FIXME: implement
|
||||
|
||||
virtual std::string type() const = 0;
|
||||
|
||||
virtual ~Input() { }
|
||||
|
||||
virtual bool operator ==(const Input & other) const { return false; }
|
||||
|
||||
/* Check whether this is a "direct" input, that is, not
|
||||
one that goes through a registry. */
|
||||
virtual bool isDirect() const { return true; }
|
||||
|
||||
/* Check whether this is an "immutable" input, that is,
|
||||
one that contains a commit hash or content hash. */
|
||||
virtual bool isImmutable() const { return (bool) narHash; }
|
||||
|
||||
virtual bool contains(const Input & other) const { return false; }
|
||||
|
||||
virtual std::optional<std::string> getRef() const { return {}; }
|
||||
|
||||
virtual std::optional<Hash> getRev() const { return {}; }
|
||||
|
||||
virtual ParsedURL toURL() const = 0;
|
||||
|
||||
std::string to_string() const
|
||||
{
|
||||
return toURL().to_string();
|
||||
}
|
||||
|
||||
Attrs toAttrs() const;
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
|
||||
|
||||
virtual std::shared_ptr<const Input> applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const;
|
||||
|
||||
virtual std::optional<Path> getSourcePath() const { return {}; }
|
||||
|
||||
virtual void markChangedFile(
|
||||
std::string_view file,
|
||||
std::optional<std::string> commitMsg) const
|
||||
{ assert(false); }
|
||||
|
||||
virtual void clone(const Path & destDir) const
|
||||
{
|
||||
throw Error("do not know how to clone input '%s'", to_string());
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
|
||||
|
||||
virtual Attrs toAttrsInternal() const = 0;
|
||||
};
|
||||
|
||||
struct InputScheme
|
||||
{
|
||||
virtual ~InputScheme() { }
|
||||
|
||||
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
|
||||
|
||||
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
|
||||
};
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const std::string & url);
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
|
||||
|
||||
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
|
||||
|
||||
struct DownloadFileResult
|
||||
{
|
||||
StorePath storePath;
|
||||
std::string etag;
|
||||
std::string effectiveUrl;
|
||||
};
|
||||
|
||||
DownloadFileResult downloadFile(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
bool immutable);
|
||||
|
||||
Tree downloadTarball(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
bool immutable);
|
||||
|
||||
}
|
||||
|
|
@ -1,460 +0,0 @@
|
|||
#include "fetchers/fetchers.hh"
|
||||
#include "fetchers/cache.hh"
|
||||
#include "fetchers/parse.hh"
|
||||
#include "globals.hh"
|
||||
#include "tarfile.hh"
|
||||
#include "store-api.hh"
|
||||
#include "regex.hh"
|
||||
|
||||
#include <sys/time.h>
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
static std::string readHead(const Path & path)
|
||||
{
|
||||
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
|
||||
}
|
||||
|
||||
struct GitInput : Input
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
bool shallow = false;
|
||||
|
||||
GitInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "git"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const GitInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& url == other2->url
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev;
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
ParsedURL url2(url);
|
||||
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
|
||||
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) url2.query.insert_or_assign("ref", *ref);
|
||||
if (shallow) url2.query.insert_or_assign("shallow", "1");
|
||||
return url2;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
if (shallow)
|
||||
attrs.emplace("shallow", true);
|
||||
return attrs;
|
||||
}
|
||||
|
||||
void clone(const Path & destDir) const override
|
||||
{
|
||||
auto [isLocal, actualUrl] = getActualUrl();
|
||||
|
||||
Strings args = {"clone"};
|
||||
|
||||
args.push_back(actualUrl);
|
||||
|
||||
if (ref) {
|
||||
args.push_back("--branch");
|
||||
args.push_back(*ref);
|
||||
}
|
||||
|
||||
if (rev) throw Error("cloning a specific revision is not implemented");
|
||||
|
||||
args.push_back(destDir);
|
||||
|
||||
runProgram("git", true, args);
|
||||
}
|
||||
|
||||
std::shared_ptr<const Input> applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const override
|
||||
{
|
||||
if (!ref && !rev) return shared_from_this();
|
||||
|
||||
auto res = std::make_shared<GitInput>(*this);
|
||||
|
||||
if (ref) res->ref = ref;
|
||||
if (rev) res->rev = rev;
|
||||
|
||||
if (!res->ref && res->rev)
|
||||
throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string());
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<Path> getSourcePath() const override
|
||||
{
|
||||
if (url.scheme == "file" && !ref && !rev)
|
||||
return url.path;
|
||||
return {};
|
||||
}
|
||||
|
||||
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
|
||||
{
|
||||
auto sourcePath = getSourcePath();
|
||||
assert(sourcePath);
|
||||
|
||||
runProgram("git", true,
|
||||
{ "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) });
|
||||
|
||||
if (commitMsg)
|
||||
runProgram("git", true,
|
||||
{ "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl() const
|
||||
{
|
||||
// Don't clone file:// URIs (but otherwise treat them the
|
||||
// same as remote URIs, i.e. don't use the working tree or
|
||||
// HEAD).
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
|
||||
bool isLocal = url.scheme == "file" && !forceHttp;
|
||||
return {isLocal, isLocal ? url.path : url.base};
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
auto name = "source";
|
||||
|
||||
auto input = std::make_shared<GitInput>(*this);
|
||||
|
||||
assert(!rev || rev->type == htSHA1);
|
||||
|
||||
auto cacheType = shallow ? "git-shallow" : "git";
|
||||
|
||||
auto getImmutableAttrs = [&]()
|
||||
{
|
||||
return Attrs({
|
||||
{"type", cacheType},
|
||||
{"name", name},
|
||||
{"rev", input->rev->gitRev()},
|
||||
});
|
||||
};
|
||||
|
||||
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
|
||||
-> std::pair<Tree, std::shared_ptr<const Input>>
|
||||
{
|
||||
assert(input->rev);
|
||||
assert(!rev || rev == input->rev);
|
||||
return {
|
||||
Tree {
|
||||
.actualPath = store->toRealPath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
|
||||
.lastModified = getIntAttr(infoAttrs, "lastModified"),
|
||||
},
|
||||
},
|
||||
input
|
||||
};
|
||||
};
|
||||
|
||||
if (rev) {
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
|
||||
auto [isLocal, actualUrl_] = getActualUrl();
|
||||
auto actualUrl = actualUrl_; // work around clang bug
|
||||
|
||||
// If this is a local directory and no ref or revision is
|
||||
// given, then allow the use of an unclean working tree.
|
||||
if (!input->ref && !input->rev && isLocal) {
|
||||
bool clean = false;
|
||||
|
||||
/* Check whether this repo has any commits. There are
|
||||
probably better ways to do this. */
|
||||
auto gitDir = actualUrl + "/.git";
|
||||
auto commonGitDir = chomp(runProgram(
|
||||
"git",
|
||||
true,
|
||||
{ "-C", actualUrl, "rev-parse", "--git-common-dir" }
|
||||
));
|
||||
if (commonGitDir != ".git")
|
||||
gitDir = commonGitDir;
|
||||
|
||||
bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
|
||||
|
||||
try {
|
||||
if (haveCommits) {
|
||||
runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
|
||||
clean = true;
|
||||
}
|
||||
} catch (ExecError & e) {
|
||||
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
|
||||
}
|
||||
|
||||
if (!clean) {
|
||||
|
||||
/* This is an unclean working tree. So copy all tracked files. */
|
||||
|
||||
if (!settings.allowDirty)
|
||||
throw Error("Git tree '%s' is dirty", actualUrl);
|
||||
|
||||
if (settings.warnDirty)
|
||||
warn("Git tree '%s' is dirty", actualUrl);
|
||||
|
||||
auto files = tokenizeString<std::set<std::string>>(
|
||||
runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s);
|
||||
|
||||
PathFilter filter = [&](const Path & p) -> bool {
|
||||
assert(hasPrefix(p, actualUrl));
|
||||
std::string file(p, actualUrl.size() + 1);
|
||||
|
||||
auto st = lstat(p);
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
auto prefix = file + "/";
|
||||
auto i = files.lower_bound(prefix);
|
||||
return i != files.end() && hasPrefix(*i, prefix);
|
||||
}
|
||||
|
||||
return files.count(file);
|
||||
};
|
||||
|
||||
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
|
||||
|
||||
auto tree = Tree {
|
||||
.actualPath = store->printStorePath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
// FIXME: maybe we should use the timestamp of the last
|
||||
// modified dirty file?
|
||||
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
|
||||
}
|
||||
};
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
}
|
||||
|
||||
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
|
||||
|
||||
Attrs mutableAttrs({
|
||||
{"type", cacheType},
|
||||
{"name", name},
|
||||
{"url", actualUrl},
|
||||
{"ref", *input->ref},
|
||||
});
|
||||
|
||||
Path repoDir;
|
||||
|
||||
if (isLocal) {
|
||||
|
||||
if (!input->rev)
|
||||
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
|
||||
|
||||
repoDir = actualUrl;
|
||||
|
||||
} else {
|
||||
|
||||
if (auto res = getCache()->lookup(store, mutableAttrs)) {
|
||||
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
|
||||
if (!rev || rev == rev2) {
|
||||
input->rev = rev2;
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
}
|
||||
|
||||
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
|
||||
repoDir = cacheDir;
|
||||
|
||||
if (!pathExists(cacheDir)) {
|
||||
createDirs(dirOf(cacheDir));
|
||||
runProgram("git", true, { "init", "--bare", repoDir });
|
||||
}
|
||||
|
||||
Path localRefFile =
|
||||
input->ref->compare(0, 5, "refs/") == 0
|
||||
? cacheDir + "/" + *input->ref
|
||||
: cacheDir + "/refs/heads/" + *input->ref;
|
||||
|
||||
bool doFetch;
|
||||
time_t now = time(0);
|
||||
|
||||
/* If a rev was specified, we need to fetch if it's not in the
|
||||
repo. */
|
||||
if (input->rev) {
|
||||
try {
|
||||
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
|
||||
doFetch = false;
|
||||
} catch (ExecError & e) {
|
||||
if (WIFEXITED(e.status)) {
|
||||
doFetch = true;
|
||||
} else {
|
||||
throw;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
|
||||
git fetch to update the local ref to the remote ref. */
|
||||
struct stat st;
|
||||
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
|
||||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
|
||||
}
|
||||
|
||||
if (doFetch) {
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
|
||||
|
||||
// FIXME: git stderr messes up our progress indicator, so
|
||||
// we're using --quiet for now. Should process its stderr.
|
||||
try {
|
||||
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) });
|
||||
} catch (Error & e) {
|
||||
if (!pathExists(localRefFile)) throw;
|
||||
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
|
||||
}
|
||||
|
||||
struct timeval times[2];
|
||||
times[0].tv_sec = now;
|
||||
times[0].tv_usec = 0;
|
||||
times[1].tv_sec = now;
|
||||
times[1].tv_usec = 0;
|
||||
|
||||
utimes(localRefFile.c_str(), times);
|
||||
}
|
||||
|
||||
if (!input->rev)
|
||||
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
|
||||
}
|
||||
|
||||
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
|
||||
|
||||
if (isShallow && !shallow)
|
||||
throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
|
||||
|
||||
// FIXME: check whether rev is an ancestor of ref.
|
||||
|
||||
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
|
||||
|
||||
/* Now that we know the ref, check again whether we have it in
|
||||
the store. */
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
|
||||
// FIXME: should pipe this, or find some better way to extract a
|
||||
// revision.
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
|
||||
gitOptions.standardOut = &sink;
|
||||
runProgram2(gitOptions);
|
||||
});
|
||||
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
|
||||
unpackTarfile(*source, tmpDir);
|
||||
|
||||
auto storePath = store->addToStore(name, tmpDir);
|
||||
|
||||
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"lastModified", lastModified},
|
||||
});
|
||||
|
||||
if (!shallow)
|
||||
infoAttrs.insert_or_assign("revCount",
|
||||
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
|
||||
|
||||
if (!this->rev)
|
||||
getCache()->add(
|
||||
store,
|
||||
mutableAttrs,
|
||||
infoAttrs,
|
||||
storePath,
|
||||
false);
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
getImmutableAttrs(),
|
||||
infoAttrs,
|
||||
storePath,
|
||||
true);
|
||||
|
||||
return makeResult(infoAttrs, std::move(storePath));
|
||||
}
|
||||
};
|
||||
|
||||
struct GitInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "git" &&
|
||||
url.scheme != "git+http" &&
|
||||
url.scheme != "git+https" &&
|
||||
url.scheme != "git+ssh" &&
|
||||
url.scheme != "git+file") return nullptr;
|
||||
|
||||
auto url2(url);
|
||||
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
|
||||
url2.query.clear();
|
||||
|
||||
Attrs attrs;
|
||||
attrs.emplace("type", "git");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "git") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow")
|
||||
throw Error("unsupported Git input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (!std::regex_match(*ref, refRegex))
|
||||
throw BadURL("invalid Git branch/tag name '%s'", *ref);
|
||||
input->ref = *ref;
|
||||
}
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
|
||||
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
|
||||
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
|
||||
|
||||
}
|
||||
|
|
@ -1,218 +0,0 @@
|
|||
#include "download.hh"
|
||||
#include "fetchers/cache.hh"
|
||||
#include "fetchers/fetchers.hh"
|
||||
#include "fetchers/parse.hh"
|
||||
#include "fetchers/regex.hh"
|
||||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
|
||||
struct GitHubInput : Input
|
||||
{
|
||||
std::string owner;
|
||||
std::string repo;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
|
||||
std::string type() const override { return "github"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const GitHubInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& owner == other2->owner
|
||||
&& repo == other2->repo
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev;
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
auto path = owner + "/" + repo;
|
||||
assert(!(ref && rev));
|
||||
if (ref) path += "/" + *ref;
|
||||
if (rev) path += "/" + rev->to_string(Base16, false);
|
||||
return ParsedURL {
|
||||
.scheme = "github",
|
||||
.path = path,
|
||||
};
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("owner", owner);
|
||||
attrs.emplace("repo", repo);
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
return attrs;
|
||||
}
|
||||
|
||||
void clone(const Path & destDir) const override
|
||||
{
|
||||
std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo));
|
||||
input = input->applyOverrides(ref.value_or("master"), rev);
|
||||
input->clone(destDir);
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
auto rev = this->rev;
|
||||
auto ref = this->ref.value_or("master");
|
||||
|
||||
if (!rev) {
|
||||
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
|
||||
owner, repo, ref);
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
rev = Hash(json["sha"], htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev->gitRev());
|
||||
}
|
||||
|
||||
auto input = std::make_shared<GitHubInput>(*this);
|
||||
input->ref = {};
|
||||
input->rev = *rev;
|
||||
|
||||
Attrs immutableAttrs({
|
||||
{"type", "git-tarball"},
|
||||
{"rev", rev->gitRev()},
|
||||
});
|
||||
|
||||
if (auto res = getCache()->lookup(store, immutableAttrs)) {
|
||||
return {
|
||||
Tree{
|
||||
.actualPath = store->toRealPath(res->second),
|
||||
.storePath = std::move(res->second),
|
||||
.info = TreeInfo {
|
||||
.lastModified = getIntAttr(res->first, "lastModified"),
|
||||
},
|
||||
},
|
||||
input
|
||||
};
|
||||
}
|
||||
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
|
||||
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
|
||||
owner, repo, rev->to_string(Base16, false));
|
||||
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;
|
||||
|
||||
auto tree = downloadTarball(store, url, "source", true);
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
immutableAttrs,
|
||||
{
|
||||
{"rev", rev->gitRev()},
|
||||
{"lastModified", *tree.info.lastModified}
|
||||
},
|
||||
tree.storePath,
|
||||
true);
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
|
||||
std::shared_ptr<const Input> applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const override
|
||||
{
|
||||
if (!ref && !rev) return shared_from_this();
|
||||
|
||||
auto res = std::make_shared<GitHubInput>(*this);
|
||||
|
||||
if (ref) res->ref = ref;
|
||||
if (rev) res->rev = rev;
|
||||
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
struct GitHubInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "github") return nullptr;
|
||||
|
||||
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
|
||||
auto input = std::make_unique<GitHubInput>();
|
||||
|
||||
if (path.size() == 2) {
|
||||
} else if (path.size() == 3) {
|
||||
if (std::regex_match(path[2], revRegex))
|
||||
input->rev = Hash(path[2], htSHA1);
|
||||
else if (std::regex_match(path[2], refRegex))
|
||||
input->ref = path[2];
|
||||
else
|
||||
throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
|
||||
} else
|
||||
throw BadURL("GitHub URL '%s' is invalid", url.url);
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev") {
|
||||
if (input->rev)
|
||||
throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
|
||||
input->rev = Hash(value, htSHA1);
|
||||
}
|
||||
else if (name == "ref") {
|
||||
if (!std::regex_match(value, refRegex))
|
||||
throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
|
||||
if (input->ref)
|
||||
throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
|
||||
input->ref = value;
|
||||
}
|
||||
}
|
||||
|
||||
if (input->ref && input->rev)
|
||||
throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
|
||||
|
||||
input->owner = path[0];
|
||||
input->repo = path[1];
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "github") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
|
||||
throw Error("unsupported GitHub input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<GitHubInput>();
|
||||
input->owner = getStrAttr(attrs, "owner");
|
||||
input->repo = getStrAttr(attrs, "repo");
|
||||
input->ref = maybeGetStrAttr(attrs, "ref");
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
|
||||
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
#include "fetchers.hh"
|
||||
#include "parse.hh"
|
||||
#include "regex.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
|
||||
struct IndirectInput : Input
|
||||
{
|
||||
std::string id;
|
||||
std::optional<Hash> rev;
|
||||
std::optional<std::string> ref;
|
||||
|
||||
std::string type() const override { return "indirect"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const IndirectInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& id == other2->id
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
}
|
||||
|
||||
bool isDirect() const override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
bool contains(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const IndirectInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& id == other2->id
|
||||
&& (!ref || ref == other2->ref)
|
||||
&& (!rev || rev == other2->rev);
|
||||
}
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
ParsedURL url;
|
||||
url.scheme = "flake";
|
||||
url.path = id;
|
||||
if (ref) { url.path += '/'; url.path += *ref; };
|
||||
if (rev) { url.path += '/'; url.path += rev->gitRev(); };
|
||||
return url;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("id", id);
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::shared_ptr<const Input> applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const override
|
||||
{
|
||||
if (!ref && !rev) return shared_from_this();
|
||||
|
||||
auto res = std::make_shared<IndirectInput>(*this);
|
||||
|
||||
if (ref) res->ref = ref;
|
||||
if (rev) res->rev = rev;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
throw Error("indirect input '%s' cannot be fetched directly", to_string());
|
||||
}
|
||||
};
|
||||
|
||||
struct IndirectInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "flake") return nullptr;
|
||||
|
||||
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
|
||||
auto input = std::make_unique<IndirectInput>();
|
||||
|
||||
if (path.size() == 1) {
|
||||
} else if (path.size() == 2) {
|
||||
if (std::regex_match(path[1], revRegex))
|
||||
input->rev = Hash(path[1], htSHA1);
|
||||
else if (std::regex_match(path[1], refRegex))
|
||||
input->ref = path[1];
|
||||
else
|
||||
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
|
||||
} else if (path.size() == 3) {
|
||||
if (!std::regex_match(path[1], refRegex))
|
||||
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
|
||||
input->ref = path[1];
|
||||
if (!std::regex_match(path[2], revRegex))
|
||||
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
|
||||
input->rev = Hash(path[2], htSHA1);
|
||||
} else
|
||||
throw BadURL("GitHub URL '%s' is invalid", url.url);
|
||||
|
||||
// FIXME: forbid query params?
|
||||
|
||||
input->id = path[0];
|
||||
if (!std::regex_match(input->id, flakeRegex))
|
||||
throw BadURL("'%s' is not a valid flake ID", input->id);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "id" && name != "ref" && name != "rev")
|
||||
throw Error("unsupported indirect input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<IndirectInput>();
|
||||
input->id = getStrAttr(attrs, "id");
|
||||
input->ref = maybeGetStrAttr(attrs, "ref");
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
|
||||
|
||||
}
|
||||
|
|
@ -1,340 +0,0 @@
|
|||
#include "fetchers/fetchers.hh"
|
||||
#include "fetchers/cache.hh"
|
||||
#include "fetchers/parse.hh"
|
||||
#include "globals.hh"
|
||||
#include "tarfile.hh"
|
||||
#include "store-api.hh"
|
||||
#include "regex.hh"
|
||||
|
||||
#include <sys/time.h>
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct MercurialInput : Input
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
|
||||
MercurialInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "hg"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const MercurialInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& url == other2->url
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev;
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
ParsedURL url2(url);
|
||||
url2.scheme = "hg+" + url2.scheme;
|
||||
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) url2.query.insert_or_assign("ref", *ref);
|
||||
return url;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::shared_ptr<const Input> applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const override
|
||||
{
|
||||
if (!ref && !rev) return shared_from_this();
|
||||
|
||||
auto res = std::make_shared<MercurialInput>(*this);
|
||||
|
||||
if (ref) res->ref = ref;
|
||||
if (rev) res->rev = rev;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<Path> getSourcePath() const
|
||||
{
|
||||
if (url.scheme == "file" && !ref && !rev)
|
||||
return url.path;
|
||||
return {};
|
||||
}
|
||||
|
||||
void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override
|
||||
{
|
||||
auto sourcePath = getSourcePath();
|
||||
assert(sourcePath);
|
||||
|
||||
// FIXME: shut up if file is already tracked.
|
||||
runProgram("hg", true,
|
||||
{ "add", *sourcePath + "/" + std::string(file) });
|
||||
|
||||
if (commitMsg)
|
||||
runProgram("hg", true,
|
||||
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl() const
|
||||
{
|
||||
bool isLocal = url.scheme == "file";
|
||||
return {isLocal, isLocal ? url.path : url.base};
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
auto name = "source";
|
||||
|
||||
auto input = std::make_shared<MercurialInput>(*this);
|
||||
|
||||
auto [isLocal, actualUrl_] = getActualUrl();
|
||||
auto actualUrl = actualUrl_; // work around clang bug
|
||||
|
||||
// FIXME: return lastModified.
|
||||
|
||||
// FIXME: don't clone local repositories.
|
||||
|
||||
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
|
||||
|
||||
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
|
||||
|
||||
if (!clean) {
|
||||
|
||||
/* This is an unclean working tree. So copy all tracked
|
||||
files. */
|
||||
|
||||
if (!settings.allowDirty)
|
||||
throw Error("Mercurial tree '%s' is unclean", actualUrl);
|
||||
|
||||
if (settings.warnDirty)
|
||||
warn("Mercurial tree '%s' is unclean", actualUrl);
|
||||
|
||||
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
|
||||
|
||||
auto files = tokenizeString<std::set<std::string>>(
|
||||
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
|
||||
|
||||
PathFilter filter = [&](const Path & p) -> bool {
|
||||
assert(hasPrefix(p, actualUrl));
|
||||
std::string file(p, actualUrl.size() + 1);
|
||||
|
||||
auto st = lstat(p);
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
auto prefix = file + "/";
|
||||
auto i = files.lower_bound(prefix);
|
||||
return i != files.end() && hasPrefix(*i, prefix);
|
||||
}
|
||||
|
||||
return files.count(file);
|
||||
};
|
||||
|
||||
auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter);
|
||||
|
||||
return {Tree {
|
||||
.actualPath = store->printStorePath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
}, input};
|
||||
}
|
||||
}
|
||||
|
||||
if (!input->ref) input->ref = "default";
|
||||
|
||||
auto getImmutableAttrs = [&]()
|
||||
{
|
||||
return Attrs({
|
||||
{"type", "hg"},
|
||||
{"name", name},
|
||||
{"rev", input->rev->gitRev()},
|
||||
});
|
||||
};
|
||||
|
||||
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
|
||||
-> std::pair<Tree, std::shared_ptr<const Input>>
|
||||
{
|
||||
assert(input->rev);
|
||||
assert(!rev || rev == input->rev);
|
||||
return {
|
||||
Tree{
|
||||
.actualPath = store->toRealPath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
.revCount = getIntAttr(infoAttrs, "revCount"),
|
||||
},
|
||||
},
|
||||
input
|
||||
};
|
||||
};
|
||||
|
||||
if (input->rev) {
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
|
||||
assert(input->rev || input->ref);
|
||||
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
|
||||
|
||||
Attrs mutableAttrs({
|
||||
{"type", "hg"},
|
||||
{"name", name},
|
||||
{"url", actualUrl},
|
||||
{"ref", *input->ref},
|
||||
});
|
||||
|
||||
if (auto res = getCache()->lookup(store, mutableAttrs)) {
|
||||
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
|
||||
if (!rev || rev == rev2) {
|
||||
input->rev = rev2;
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
}
|
||||
|
||||
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
|
||||
|
||||
/* If this is a commit hash that we already have, we don't
|
||||
have to pull again. */
|
||||
if (!(input->rev
|
||||
&& pathExists(cacheDir)
|
||||
&& runProgram(
|
||||
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
|
||||
.killStderr(true)).second == "1"))
|
||||
{
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
|
||||
|
||||
if (pathExists(cacheDir)) {
|
||||
try {
|
||||
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
|
||||
}
|
||||
catch (ExecError & e) {
|
||||
string transJournal = cacheDir + "/.hg/store/journal";
|
||||
/* hg throws "abandoned transaction" error only if this file exists */
|
||||
if (pathExists(transJournal)) {
|
||||
runProgram("hg", true, { "recover", "-R", cacheDir });
|
||||
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
|
||||
} else {
|
||||
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
createDirs(dirOf(cacheDir));
|
||||
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
|
||||
}
|
||||
}
|
||||
|
||||
auto tokens = tokenizeString<std::vector<std::string>>(
|
||||
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
|
||||
assert(tokens.size() == 3);
|
||||
|
||||
input->rev = Hash(tokens[0], htSHA1);
|
||||
auto revCount = std::stoull(tokens[1]);
|
||||
input->ref = tokens[2];
|
||||
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
|
||||
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
|
||||
|
||||
deletePath(tmpDir + "/.hg_archival.txt");
|
||||
|
||||
auto storePath = store->addToStore(name, tmpDir);
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"revCount", (int64_t) revCount},
|
||||
});
|
||||
|
||||
if (!this->rev)
|
||||
getCache()->add(
|
||||
store,
|
||||
mutableAttrs,
|
||||
infoAttrs,
|
||||
storePath,
|
||||
false);
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
getImmutableAttrs(),
|
||||
infoAttrs,
|
||||
storePath,
|
||||
true);
|
||||
|
||||
return makeResult(infoAttrs, std::move(storePath));
|
||||
}
|
||||
};
|
||||
|
||||
struct MercurialInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "hg+http" &&
|
||||
url.scheme != "hg+https" &&
|
||||
url.scheme != "hg+ssh" &&
|
||||
url.scheme != "hg+file") return nullptr;
|
||||
|
||||
auto url2(url);
|
||||
url2.scheme = std::string(url2.scheme, 3);
|
||||
url2.query.clear();
|
||||
|
||||
Attrs attrs;
|
||||
attrs.emplace("type", "hg");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev")
|
||||
throw Error("unsupported Mercurial input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (!std::regex_match(*ref, refRegex))
|
||||
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
|
||||
input->ref = *ref;
|
||||
}
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
|
||||
|
||||
}
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
#include "parse.hh"
|
||||
#include "util.hh"
|
||||
#include "regex.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::regex refRegex(refRegexS, std::regex::ECMAScript);
|
||||
std::regex revRegex(revRegexS, std::regex::ECMAScript);
|
||||
std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
|
||||
|
||||
ParsedURL parseURL(const std::string & url)
|
||||
{
|
||||
static std::regex uriRegex(
|
||||
"((" + schemeRegex + "):"
|
||||
+ "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
|
||||
+ "(?:\\?(" + queryRegex + "))?"
|
||||
+ "(?:#(" + queryRegex + "))?",
|
||||
std::regex::ECMAScript);
|
||||
|
||||
std::smatch match;
|
||||
|
||||
if (std::regex_match(url, match, uriRegex)) {
|
||||
auto & base = match[1];
|
||||
std::string scheme = match[2];
|
||||
auto authority = match[3].matched
|
||||
? std::optional<std::string>(match[3]) : std::nullopt;
|
||||
std::string path = match[4].matched ? match[4] : match[5];
|
||||
auto & query = match[6];
|
||||
auto & fragment = match[7];
|
||||
|
||||
auto isFile = scheme.find("file") != std::string::npos;
|
||||
|
||||
if (authority && *authority != "" && isFile)
|
||||
throw Error("file:// URL '%s' has unexpected authority '%s'",
|
||||
url, *authority);
|
||||
|
||||
if (isFile && path.empty())
|
||||
path = "/";
|
||||
|
||||
return ParsedURL{
|
||||
.url = url,
|
||||
.base = base,
|
||||
.scheme = scheme,
|
||||
.authority = authority,
|
||||
.path = path,
|
||||
.query = decodeQuery(query),
|
||||
.fragment = percentDecode(std::string(fragment))
|
||||
};
|
||||
}
|
||||
|
||||
else
|
||||
throw BadURL("'%s' is not a valid URL", url);
|
||||
}
|
||||
|
||||
std::string percentDecode(std::string_view in)
|
||||
{
|
||||
std::string decoded;
|
||||
for (size_t i = 0; i < in.size(); ) {
|
||||
if (in[i] == '%') {
|
||||
if (i + 2 >= in.size())
|
||||
throw BadURL("invalid URI parameter '%s'", in);
|
||||
try {
|
||||
decoded += std::stoul(std::string(in, i + 1, 2), 0, 16);
|
||||
i += 3;
|
||||
} catch (...) {
|
||||
throw BadURL("invalid URI parameter '%s'", in);
|
||||
}
|
||||
} else
|
||||
decoded += in[i++];
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> decodeQuery(const std::string & query)
|
||||
{
|
||||
std::map<std::string, std::string> result;
|
||||
|
||||
for (auto s : tokenizeString<Strings>(query, "&")) {
|
||||
auto e = s.find('=');
|
||||
if (e != std::string::npos)
|
||||
result.emplace(
|
||||
s.substr(0, e),
|
||||
percentDecode(std::string_view(s).substr(e + 1)));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string percentEncode(std::string_view s)
|
||||
{
|
||||
std::string res;
|
||||
for (auto & c : s)
|
||||
if ((c >= 'a' && c <= 'z')
|
||||
|| (c >= 'A' && c <= 'Z')
|
||||
|| (c >= '0' && c <= '9')
|
||||
|| strchr("-._~!$&'()*+,;=:@", c))
|
||||
res += c;
|
||||
else
|
||||
res += fmt("%%%02x", (unsigned int) c);
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string encodeQuery(const std::map<std::string, std::string> & ss)
|
||||
{
|
||||
std::string res;
|
||||
bool first = true;
|
||||
for (auto & [name, value] : ss) {
|
||||
if (!first) res += '&';
|
||||
first = false;
|
||||
res += percentEncode(name);
|
||||
res += '=';
|
||||
res += percentEncode(value);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string ParsedURL::to_string() const
|
||||
{
|
||||
return
|
||||
scheme
|
||||
+ ":"
|
||||
+ (authority ? "//" + *authority : "")
|
||||
+ path
|
||||
+ (query.empty() ? "" : "?" + encodeQuery(query))
|
||||
+ (fragment.empty() ? "" : "#" + percentEncode(fragment));
|
||||
}
|
||||
|
||||
bool ParsedURL::operator ==(const ParsedURL & other) const
|
||||
{
|
||||
return
|
||||
scheme == other.scheme
|
||||
&& authority == other.authority
|
||||
&& path == other.path
|
||||
&& query == other.query
|
||||
&& fragment == other.fragment;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct ParsedURL
|
||||
{
|
||||
std::string url;
|
||||
std::string base; // URL without query/fragment
|
||||
std::string scheme;
|
||||
std::optional<std::string> authority;
|
||||
std::string path;
|
||||
std::map<std::string, std::string> query;
|
||||
std::string fragment;
|
||||
|
||||
std::string to_string() const;
|
||||
|
||||
bool operator ==(const ParsedURL & other) const;
|
||||
};
|
||||
|
||||
MakeError(BadURL, Error);
|
||||
|
||||
std::string percentDecode(std::string_view in);
|
||||
|
||||
std::map<std::string, std::string> decodeQuery(const std::string & query);
|
||||
|
||||
ParsedURL parseURL(const std::string & url);
|
||||
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <regex>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
// URI stuff.
|
||||
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
|
||||
const static std::string schemeRegex = "(?:[a-z+]+)";
|
||||
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
|
||||
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
|
||||
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
|
||||
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
|
||||
const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
|
||||
const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
|
||||
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
|
||||
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
|
||||
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
|
||||
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
|
||||
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
|
||||
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
|
||||
|
||||
// A Git ref (i.e. branch or tag name).
|
||||
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
|
||||
extern std::regex refRegex;
|
||||
|
||||
// A Git revision (a SHA-1 commit hash).
|
||||
const static std::string revRegexS = "[0-9a-fA-F]{40}";
|
||||
extern std::regex revRegex;
|
||||
|
||||
// A ref or revision, or a ref followed by a revision.
|
||||
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
|
||||
|
||||
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
|
||||
extern std::regex flakeIdRegex;
|
||||
|
||||
}
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
#include "fetchers/registry.hh"
|
||||
#include "fetchers/fetchers.hh"
|
||||
#include "util.hh"
|
||||
#include "globals.hh"
|
||||
#include "download.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::shared_ptr<Registry> Registry::read(
|
||||
const Path & path, RegistryType type)
|
||||
{
|
||||
auto registry = std::make_shared<Registry>(type);
|
||||
|
||||
if (!pathExists(path))
|
||||
return std::make_shared<Registry>(type);
|
||||
|
||||
auto json = nlohmann::json::parse(readFile(path));
|
||||
|
||||
auto version = json.value("version", 0);
|
||||
|
||||
// FIXME: remove soon
|
||||
if (version == 1) {
|
||||
auto flakes = json["flakes"];
|
||||
for (auto i = flakes.begin(); i != flakes.end(); ++i) {
|
||||
auto url = i->value("url", i->value("uri", ""));
|
||||
if (url.empty())
|
||||
throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'",
|
||||
path, i.key());
|
||||
registry->entries.push_back(
|
||||
{inputFromURL(i.key()), inputFromURL(url), {}});
|
||||
}
|
||||
}
|
||||
|
||||
else if (version == 2) {
|
||||
for (auto & i : json["flakes"]) {
|
||||
auto toAttrs = jsonToAttrs(i["to"]);
|
||||
Attrs extraAttrs;
|
||||
auto j = toAttrs.find("dir");
|
||||
if (j != toAttrs.end()) {
|
||||
extraAttrs.insert(*j);
|
||||
toAttrs.erase(j);
|
||||
}
|
||||
registry->entries.push_back(
|
||||
{ inputFromAttrs(jsonToAttrs(i["from"]))
|
||||
, inputFromAttrs(toAttrs)
|
||||
, extraAttrs
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
else
|
||||
throw Error("flake registry '%s' has unsupported version %d", path, version);
|
||||
|
||||
|
||||
return registry;
|
||||
}
|
||||
|
||||
void Registry::write(const Path & path)
|
||||
{
|
||||
nlohmann::json arr;
|
||||
for (auto & elem : entries) {
|
||||
nlohmann::json obj;
|
||||
obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs());
|
||||
obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs());
|
||||
if (!std::get<2>(elem).empty())
|
||||
obj["to"].update(attrsToJson(std::get<2>(elem)));
|
||||
arr.emplace_back(std::move(obj));
|
||||
}
|
||||
|
||||
nlohmann::json json;
|
||||
json["version"] = 2;
|
||||
json["flakes"] = std::move(arr);
|
||||
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, json.dump(2));
|
||||
}
|
||||
|
||||
void Registry::add(
|
||||
const std::shared_ptr<const Input> & from,
|
||||
const std::shared_ptr<const Input> & to,
|
||||
const Attrs & extraAttrs)
|
||||
{
|
||||
entries.emplace_back(from, to, extraAttrs);
|
||||
}
|
||||
|
||||
void Registry::remove(const std::shared_ptr<const Input> & input)
|
||||
{
|
||||
// FIXME: use C++20 std::erase.
|
||||
for (auto i = entries.begin(); i != entries.end(); )
|
||||
if (*std::get<0>(*i) == *input)
|
||||
i = entries.erase(i);
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
Path getUserRegistryPath()
|
||||
{
|
||||
return getHome() + "/.config/nix/registry.json";
|
||||
}
|
||||
|
||||
std::shared_ptr<Registry> getUserRegistry()
|
||||
{
|
||||
return Registry::read(getUserRegistryPath(), Registry::User);
|
||||
}
|
||||
|
||||
static std::shared_ptr<Registry> flagRegistry =
|
||||
std::make_shared<Registry>(Registry::Flag);
|
||||
|
||||
std::shared_ptr<Registry> getFlagRegistry()
|
||||
{
|
||||
return flagRegistry;
|
||||
}
|
||||
|
||||
void overrideRegistry(
|
||||
const std::shared_ptr<const Input> & from,
|
||||
const std::shared_ptr<const Input> & to,
|
||||
const Attrs & extraAttrs)
|
||||
{
|
||||
flagRegistry->add(from, to, extraAttrs);
|
||||
}
|
||||
|
||||
static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
|
||||
{
|
||||
static auto reg = [&]() {
|
||||
auto path = settings.flakeRegistry;
|
||||
|
||||
if (!hasPrefix(path, "/")) {
|
||||
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
|
||||
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
|
||||
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
|
||||
path = store->toRealPath(storePath);
|
||||
}
|
||||
|
||||
return Registry::read(path, Registry::Global);
|
||||
}();
|
||||
|
||||
return reg;
|
||||
}
|
||||
|
||||
Registries getRegistries(ref<Store> store)
|
||||
{
|
||||
Registries registries;
|
||||
registries.push_back(getFlagRegistry());
|
||||
registries.push_back(getUserRegistry());
|
||||
registries.push_back(getGlobalRegistry(store));
|
||||
return registries;
|
||||
}
|
||||
|
||||
std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
|
||||
ref<Store> store,
|
||||
std::shared_ptr<const Input> input)
|
||||
{
|
||||
Attrs extraAttrs;
|
||||
int n = 0;
|
||||
|
||||
restart:
|
||||
|
||||
n++;
|
||||
if (n > 100) throw Error("cycle detected in flake registr for '%s'", input);
|
||||
|
||||
for (auto & registry : getRegistries(store)) {
|
||||
// FIXME: O(n)
|
||||
for (auto & entry : registry->entries) {
|
||||
auto from = std::get<0>(entry);
|
||||
if (from->contains(*input)) {
|
||||
input = std::get<1>(entry)->applyOverrides(
|
||||
!from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(),
|
||||
!from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>());
|
||||
extraAttrs = std::get<2>(entry);
|
||||
goto restart;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!input->isDirect())
|
||||
throw Error("cannot find flake '%s' in the flake registries", input->to_string());
|
||||
|
||||
return {input, extraAttrs};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "fetchers.hh"
|
||||
|
||||
namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Registry
|
||||
{
|
||||
enum RegistryType {
|
||||
Flag = 0,
|
||||
User = 1,
|
||||
Global = 2,
|
||||
};
|
||||
|
||||
RegistryType type;
|
||||
|
||||
std::vector<
|
||||
std::tuple<
|
||||
std::shared_ptr<const Input>, // from
|
||||
std::shared_ptr<const Input>, // to
|
||||
Attrs // extra attributes
|
||||
>
|
||||
> entries;
|
||||
|
||||
Registry(RegistryType type)
|
||||
: type(type)
|
||||
{ }
|
||||
|
||||
static std::shared_ptr<Registry> read(
|
||||
const Path & path, RegistryType type);
|
||||
|
||||
void write(const Path & path);
|
||||
|
||||
void add(
|
||||
const std::shared_ptr<const Input> & from,
|
||||
const std::shared_ptr<const Input> & to,
|
||||
const Attrs & extraAttrs);
|
||||
|
||||
void remove(const std::shared_ptr<const Input> & input);
|
||||
};
|
||||
|
||||
typedef std::vector<std::shared_ptr<Registry>> Registries;
|
||||
|
||||
std::shared_ptr<Registry> getUserRegistry();
|
||||
|
||||
Path getUserRegistryPath();
|
||||
|
||||
Registries getRegistries(ref<Store> store);
|
||||
|
||||
void overrideRegistry(
|
||||
const std::shared_ptr<const Input> & from,
|
||||
const std::shared_ptr<const Input> & to,
|
||||
const Attrs & extraAttrs);
|
||||
|
||||
std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries(
|
||||
ref<Store> store,
|
||||
std::shared_ptr<const Input> input);
|
||||
|
||||
}
|
||||
|
|
@ -1,278 +0,0 @@
|
|||
#include "fetchers/fetchers.hh"
|
||||
#include "fetchers/parse.hh"
|
||||
#include "fetchers/cache.hh"
|
||||
#include "download.hh"
|
||||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
#include "archive.hh"
|
||||
#include "tarfile.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
DownloadFileResult downloadFile(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
bool immutable)
|
||||
{
|
||||
// FIXME: check store
|
||||
|
||||
Attrs inAttrs({
|
||||
{"type", "file"},
|
||||
{"url", url},
|
||||
{"name", name},
|
||||
});
|
||||
|
||||
auto cached = getCache()->lookupExpired(store, inAttrs);
|
||||
|
||||
auto useCached = [&]() -> DownloadFileResult
|
||||
{
|
||||
return {
|
||||
.storePath = std::move(cached->storePath),
|
||||
.etag = getStrAttr(cached->infoAttrs, "etag"),
|
||||
.effectiveUrl = getStrAttr(cached->infoAttrs, "url")
|
||||
};
|
||||
};
|
||||
|
||||
if (cached && !cached->expired)
|
||||
return useCached();
|
||||
|
||||
DownloadRequest request(url);
|
||||
if (cached)
|
||||
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
|
||||
DownloadResult res;
|
||||
try {
|
||||
res = getDownloader()->download(request);
|
||||
} catch (DownloadError & e) {
|
||||
if (cached) {
|
||||
warn("%s; using cached version", e.msg());
|
||||
return useCached();
|
||||
} else
|
||||
throw;
|
||||
}
|
||||
|
||||
// FIXME: write to temporary file.
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"etag", res.etag},
|
||||
{"url", res.effectiveUri},
|
||||
});
|
||||
|
||||
std::optional<StorePath> storePath;
|
||||
|
||||
if (res.cached) {
|
||||
assert(cached);
|
||||
assert(request.expectedETag == res.etag);
|
||||
storePath = std::move(cached->storePath);
|
||||
} else {
|
||||
StringSink sink;
|
||||
dumpString(*res.data, sink);
|
||||
auto hash = hashString(htSHA256, *res.data);
|
||||
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
|
||||
info.narHash = hashString(htSHA256, *sink.s);
|
||||
info.narSize = sink.s->size();
|
||||
info.ca = makeFixedOutputCA(false, hash);
|
||||
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
|
||||
storePath = std::move(info.path);
|
||||
}
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
inAttrs,
|
||||
infoAttrs,
|
||||
*storePath,
|
||||
immutable);
|
||||
|
||||
if (url != res.effectiveUri)
|
||||
getCache()->add(
|
||||
store,
|
||||
{
|
||||
{"type", "file"},
|
||||
{"url", res.effectiveUri},
|
||||
{"name", name},
|
||||
},
|
||||
infoAttrs,
|
||||
*storePath,
|
||||
immutable);
|
||||
|
||||
return {
|
||||
.storePath = std::move(*storePath),
|
||||
.etag = res.etag,
|
||||
.effectiveUrl = res.effectiveUri,
|
||||
};
|
||||
}
|
||||
|
||||
Tree downloadTarball(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
bool immutable)
|
||||
{
|
||||
Attrs inAttrs({
|
||||
{"type", "tarball"},
|
||||
{"url", url},
|
||||
{"name", name},
|
||||
});
|
||||
|
||||
auto cached = getCache()->lookupExpired(store, inAttrs);
|
||||
|
||||
if (cached && !cached->expired)
|
||||
return Tree {
|
||||
.actualPath = store->toRealPath(cached->storePath),
|
||||
.storePath = std::move(cached->storePath),
|
||||
.info = TreeInfo {
|
||||
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
|
||||
},
|
||||
};
|
||||
|
||||
auto res = downloadFile(store, url, name, immutable);
|
||||
|
||||
std::optional<StorePath> unpackedStorePath;
|
||||
time_t lastModified;
|
||||
|
||||
if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
|
||||
unpackedStorePath = std::move(cached->storePath);
|
||||
lastModified = getIntAttr(cached->infoAttrs, "lastModified");
|
||||
} else {
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete autoDelete(tmpDir, true);
|
||||
unpackTarfile(store->toRealPath(res.storePath), tmpDir);
|
||||
auto members = readDirectory(tmpDir);
|
||||
if (members.size() != 1)
|
||||
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
|
||||
auto topDir = tmpDir + "/" + members.begin()->name;
|
||||
lastModified = lstat(topDir).st_mtime;
|
||||
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
|
||||
}
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"lastModified", lastModified},
|
||||
{"etag", res.etag},
|
||||
});
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
inAttrs,
|
||||
infoAttrs,
|
||||
*unpackedStorePath,
|
||||
immutable);
|
||||
|
||||
return Tree {
|
||||
.actualPath = store->toRealPath(*unpackedStorePath),
|
||||
.storePath = std::move(*unpackedStorePath),
|
||||
.info = TreeInfo {
|
||||
.lastModified = lastModified,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
struct TarballInput : Input
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<Hash> hash;
|
||||
|
||||
TarballInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "tarball"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const TarballInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& to_string() == other2->to_string()
|
||||
&& hash == other2->hash;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return hash || narHash;
|
||||
}
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
auto url2(url);
|
||||
// NAR hashes are preferred over file hashes since tar/zip files
|
||||
// don't have a canonical representation.
|
||||
if (narHash)
|
||||
url2.query.insert_or_assign("narHash", narHash->to_string(SRI));
|
||||
else if (hash)
|
||||
url2.query.insert_or_assign("hash", hash->to_string(SRI));
|
||||
return url2;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (narHash)
|
||||
attrs.emplace("narHash", narHash->to_string(SRI));
|
||||
else if (hash)
|
||||
attrs.emplace("hash", hash->to_string(SRI));
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
auto tree = downloadTarball(store, url.to_string(), "source", false);
|
||||
|
||||
auto input = std::make_shared<TarballInput>(*this);
|
||||
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
};
|
||||
|
||||
struct TarballInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
|
||||
|
||||
if (!hasSuffix(url.path, ".zip")
|
||||
&& !hasSuffix(url.path, ".tar")
|
||||
&& !hasSuffix(url.path, ".tar.gz")
|
||||
&& !hasSuffix(url.path, ".tar.xz")
|
||||
&& !hasSuffix(url.path, ".tar.bz2"))
|
||||
return nullptr;
|
||||
|
||||
auto input = std::make_unique<TarballInput>(url);
|
||||
|
||||
auto hash = input->url.query.find("hash");
|
||||
if (hash != input->url.query.end()) {
|
||||
// FIXME: require SRI hash.
|
||||
input->hash = Hash(hash->second);
|
||||
input->url.query.erase(hash);
|
||||
}
|
||||
|
||||
auto narHash = input->url.query.find("narHash");
|
||||
if (narHash != input->url.query.end()) {
|
||||
// FIXME: require SRI hash.
|
||||
input->narHash = Hash(narHash->second);
|
||||
input->url.query.erase(narHash);
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "hash" && name != "narHash")
|
||||
throw Error("unsupported tarball input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto hash = maybeGetStrAttr(attrs, "hash"))
|
||||
// FIXME: require SRI hash.
|
||||
input->hash = Hash(*hash);
|
||||
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
|
||||
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct TreeInfo
|
||||
{
|
||||
Hash narHash;
|
||||
std::optional<uint64_t> revCount;
|
||||
std::optional<time_t> lastModified;
|
||||
|
||||
bool operator ==(const TreeInfo & other) const
|
||||
{
|
||||
return
|
||||
narHash == other.narHash
|
||||
&& revCount == other.revCount
|
||||
&& lastModified == other.lastModified;
|
||||
}
|
||||
|
||||
StorePath computeStorePath(Store & store) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
#include "thread-pool.hh"
|
||||
#include "json.hh"
|
||||
#include "derivations.hh"
|
||||
#include "fetchers/parse.hh"
|
||||
#include "url.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
|
|
@ -867,7 +867,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
|||
Store::Params params;
|
||||
auto q = uri.find('?');
|
||||
if (q != std::string::npos) {
|
||||
params = fetchers::decodeQuery(uri.substr(q + 1));
|
||||
params = decodeQuery(uri.substr(q + 1));
|
||||
uri = uri_.substr(0, q);
|
||||
}
|
||||
return {uri, params};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue