mirror of
https://github.com/NixOS/nix.git
synced 2025-11-12 05:26:02 +01:00
Fetch commits from github/gitlab using Auth header
`nix flake info` calls the github 'commits' API, which requires authorization when the repository is private. Currently this request fails with a 404. This commit adds an authorization header when calling the 'commits' API. It also changes the way that the 'tarball' API authenticates, moving the user's token from a query parameter into the Authorization header. The query parameter method is recently deprecated and will be disallowed in November 2020. Using them today triggers a warning email.
This commit is contained in:
Greg Hale
parent
5080d4e7b2
commit
a303c0b6dc
12 changed files with 84 additions and 31 deletions
|
|
@ -3,11 +3,24 @@
|
|||
#include "fetchers.hh"
|
||||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
#include "types.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct DownloadUrl
|
||||
{
|
||||
std::string url;
|
||||
std::optional<std::pair<std::string, std::string>> access_token_header;
|
||||
|
||||
DownloadUrl(const std::string & url)
|
||||
: url(url) { }
|
||||
|
||||
DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
|
||||
: url(url), access_token_header(access_token_header) { }
|
||||
};
|
||||
|
||||
// A github or gitlab url
|
||||
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
|
||||
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
|
||||
|
|
@ -16,6 +29,8 @@ struct GitArchiveInputScheme : InputScheme
|
|||
{
|
||||
virtual std::string type() = 0;
|
||||
|
||||
virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
|
||||
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != type()) return {};
|
||||
|
|
@ -131,7 +146,7 @@ struct GitArchiveInputScheme : InputScheme
|
|||
|
||||
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
|
||||
|
||||
virtual std::string getDownloadUrl(const Input & input) const = 0;
|
||||
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
|
||||
{
|
||||
|
|
@ -160,7 +175,12 @@ struct GitArchiveInputScheme : InputScheme
|
|||
|
||||
auto url = getDownloadUrl(input);
|
||||
|
||||
auto [tree, lastModified] = downloadTarball(store, url, "source", true);
|
||||
Headers headers;
|
||||
if (url.access_token_header) {
|
||||
headers.push_back(*url.access_token_header);
|
||||
}
|
||||
|
||||
auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
|
||||
|
||||
input.attrs.insert_or_assign("lastModified", lastModified);
|
||||
|
||||
|
|
@ -182,11 +202,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
std::string type() override { return "github"; }
|
||||
|
||||
void addAccessToken(std::string & url) const
|
||||
{
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;
|
||||
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
|
||||
return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
|
||||
}
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
|
|
@ -195,18 +212,21 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
|
||||
addAccessToken(url);
|
||||
Headers headers;
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
headers.push_back(accessHeaderFromToken(accessToken));
|
||||
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
downloadFile(store, url, headers, "source", false).storePath)));
|
||||
auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
|
||||
return rev;
|
||||
}
|
||||
|
||||
std::string getDownloadUrl(const Input & input) const override
|
||||
DownloadUrl getDownloadUrl(const Input & input) const override
|
||||
{
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
|
|
@ -215,9 +235,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
addAccessToken(url);
|
||||
|
||||
return url;
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "") {
|
||||
auto auth_header = accessHeaderFromToken(accessToken);
|
||||
return DownloadUrl(url, auth_header);
|
||||
} else {
|
||||
return DownloadUrl(url);
|
||||
}
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
|
|
@ -234,21 +258,31 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
std::string type() override { return "gitlab"; }
|
||||
|
||||
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
|
||||
return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
|
||||
}
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
|
||||
Headers headers;
|
||||
std::string accessToken = settings.gitlabAccessToken.get();
|
||||
if (accessToken != "")
|
||||
headers.push_back(accessHeaderFromToken(accessToken));
|
||||
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
downloadFile(store, url, headers, "source", false).storePath)));
|
||||
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
|
||||
return rev;
|
||||
}
|
||||
|
||||
std::string getDownloadUrl(const Input & input) const override
|
||||
DownloadUrl getDownloadUrl(const Input & input) const override
|
||||
{
|
||||
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
|
|
@ -256,12 +290,14 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
|||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;*/
|
||||
std::string accessToken = settings.gitlabAccessToken.get();
|
||||
if (accessToken != "") {
|
||||
auto auth_header = accessHeaderFromToken(accessToken);
|
||||
return DownloadUrl(url, auth_header);
|
||||
} else {
|
||||
return DownloadUrl(url);
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue