nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-20 17:29:36 +01:00
Code Activity

Merge branch 'master' into read-only-local-store

Browse source
This commit is contained in:
Ben Radford 2023-06-19 14:09:20 +01:00
commit 0b3fa995e5
No known key found for this signature in database
GPG key ID: 9DF5D4640AB888D5
42 changed files with 663 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/libstore/filetransfer.cc
View file

@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
std::string line((char *) contents, realSize);
printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
std::smatch match;
if (std::regex_match(line, match, statusLine)) {
if (std::smatch match; std::regex_match(line, match, statusLine)) {
result.etag = "";
result.data.clear();
result.bodySize = 0;
@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer
acceptRanges = false;
encoding = "";
} else {
auto i = line.find(':');
if (i != std::string::npos) {
std::string name = toLower(trim(line.substr(0, i)));
if (name == "etag") {
result.etag = trim(line.substr(i + 1));
/* Hack to work around a GitHub bug: it sends
@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer
debug("shutting down on 200 HTTP response with expected ETag");
return 0;
}
} else if (name == "content-encoding")
}
else if (name == "content-encoding")
encoding = trim(line.substr(i + 1));
else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
acceptRanges = true;
else if (name == "link" || name == "x-amz-meta-link") {
auto value = trim(line.substr(i + 1));
static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase);
if (std::smatch match; std::regex_match(value, match, linkRegex))
result.immutableUrl = match.str(1);
else
debug("got invalid link header '%s'", value);
}
}
}
return realSize;
@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer
{
auto httpStatus = getHTTPStatus();
char * effectiveUriCStr;
char * effectiveUriCStr = nullptr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr)
result.effectiveUri = effectiveUriCStr;

4
src/libstore/filetransfer.hh
View file

@ -80,6 +80,10 @@ struct FileTransferResult
std::string effectiveUri;
std::string data;
uint64_t bodySize = 0;
/* An "immutable" URL for this resource (i.e. one whose contents
will never change), as returned by the `Link: <url>;
rel="immutable"` header. */
std::optional<std::string> immutableUrl;
};
class Store;

25
src/libstore/globals.cc
View file

@ -77,7 +77,30 @@ Settings::Settings()
allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
#endif
buildHook = getSelfExe().value_or("nix") + " __build-remote";
/* Set the build hook location
For builds we perform a self-invocation, so Nix has to be self-aware.
That is, it has to know where it is installed. We don't think it's sentient.
Normally, nix is installed according to `nixBinDir`, which is set at compile time,
but can be overridden. This makes for a great default that works even if this
code is linked as a library into some other program whose main is not aware
that it might need to be a build remote hook.
However, it may not have been installed at all. For example, if it's a static build,
there's a good chance that it has been moved out of its installation directory.
That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the
current executable, using `getSelfExe()`.
As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible.
If you're porting Nix to a new platform, that might be good enough for a while, but
you'll want to improve `getSelfExe()` to work on your platform.
*/
std::string nixExePath = nixBinDir + "/nix";
if (!pathExists(nixExePath)) {
nixExePath = getSelfExe().value_or("nix");
}
buildHook = nixExePath + " __build-remote";
}
void loadConfFile()

123
src/libstore/profiles.cc
View file

@ -13,8 +13,10 @@
namespace nix {
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
/**
* Parse a generation name of the format
* `<profilename>-<number>-link'.
*/
static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name)
{
if (name.substr(0, profileName.size() + 1) != profileName + "-") return {};
@ -28,7 +30,6 @@ static std::optional<GenerationNumber> parseName(const std::string & profileName
}
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
{
Generations gens;
@ -61,15 +62,16 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
}
static void makeName(const Path & profile, GenerationNumber num,
Path & outLink)
/**
* Create a generation name that can be parsed by `parseName()`.
*/
static Path makeName(const Path & profile, GenerationNumber num)
{
Path prefix = fmt("%1%-%2%", profile, num);
outLink = prefix + "-link";
return fmt("%s-%s-link", profile, num);
}
Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath)
{
/* The new generation number should be higher than old the
previous ones. */
@ -79,7 +81,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
if (gens.size() > 0) {
Generation last = gens.back();
if (readLink(last.path) == store->printStorePath(outPath)) {
if (readLink(last.path) == store.printStorePath(outPath)) {
/* We only create a new generation symlink if it differs
from the last one.
@ -89,7 +91,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
return last.path;
}
num = gens.back().number;
num = last.number;
} else {
num = 0;
}
@ -100,9 +102,8 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
to the permanent roots (of which the GC would have a stale
view). If we didn't do it this way, the GC might remove the
user environment etc. we've just built. */
Path generation;
makeName(profile, num + 1, generation);
store->addPermRoot(outPath, generation);
Path generation = makeName(profile, num + 1);
store.addPermRoot(outPath, generation);
return generation;
}
@ -117,12 +118,19 @@ static void removeFile(const Path & path)
void deleteGeneration(const Path & profile, GenerationNumber gen)
{
Path generation;
makeName(profile, gen, generation);
Path generation = makeName(profile, gen);
removeFile(generation);
}
/**
* Delete a generation with dry-run mode.
*
* Like `deleteGeneration()` but:
*
* - We log what we are going to do.
*
* - We only actually delete if `dryRun` is false.
*/
static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
{
if (dryRun)
@ -150,27 +158,36 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
}
}
/**
* Advanced the iterator until the given predicate `cond` returns `true`.
*/
static inline void iterDropUntil(Generations & gens, auto && i, auto && cond)
{
for (; i != gens.rend() && !cond(*i); ++i);
}
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
{
if (max == 0)
throw Error("Must keep at least one generation, otherwise the current one would be deleted");
PathLocks lock;
lockProfile(lock, profile);
bool fromCurGen = false;
auto [gens, curGen] = findGenerations(profile);
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
if (i->number == curGen) {
fromCurGen = true;
max--;
continue;
}
if (fromCurGen) {
if (max) {
max--;
continue;
}
deleteGeneration2(profile, i->number, dryRun);
}
}
auto [gens, _curGen] = findGenerations(profile);
auto curGen = _curGen;
auto i = gens.rbegin();
// Find the current generation
iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; });
// Skip over `max` generations, preserving them
for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep);
// Delete the rest
for (; i != gens.rend(); ++i)
deleteGeneration2(profile, i->number, dryRun);
}
void deleteOldGenerations(const Path & profile, bool dryRun)
@ -193,23 +210,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
auto [gens, curGen] = findGenerations(profile);
bool canDelete = false;
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
if (canDelete) {
assert(i->creationTime < t);
if (i->number != curGen)
deleteGeneration2(profile, i->number, dryRun);
} else if (i->creationTime < t) {
/* We may now start deleting generations, but we don't
delete this generation yet, because this generation was
still the one that was active at the requested point in
time. */
canDelete = true;
}
auto i = gens.rbegin();
// Predicate that the generation is older than the given time.
auto older = [&](auto & g) { return g.creationTime < t; };
// Find the first older generation, if one exists
iterDropUntil(gens, i, older);
/* Take the previous generation
We don't want delete this one yet because it
existed at the requested point in time, and
we want to be able to roll back to it. */
if (i != gens.rend()) ++i;
// Delete all previous generations (unless current).
for (; i != gens.rend(); ++i) {
/* Creating date and generations should be monotonic, so lower
numbered derivations should also be older. */
assert(older(*i));
if (i->number != curGen)
deleteGeneration2(profile, i->number, dryRun);
}
}
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun)
time_t parseOlderThanTimeSpec(std::string_view timeSpec)
{
if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
@ -221,9 +248,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec,
if (!days || *days < 1)
throw UsageError("invalid number of days specifier '%1%'", timeSpec);
time_t oldTime = curTime - *days * 24 * 3600;
deleteGenerationsOlderThan(profile, oldTime, dryRun);
return curTime - *days * 24 * 3600;
}

143
src/libstore/profiles.hh
View file

@ -1,7 +1,11 @@
#pragma once
///@file
/**
* @file Implementation of Profiles.
*
* See the manual for additional information.
*/
#include "types.hh"
#include "types.hh"
#include "pathlocks.hh"
#include <time.h>
@ -12,41 +16,166 @@ namespace nix {
class StorePath;
/**
* A positive number identifying a generation for a given profile.
*
* Generation numbers are assigned sequentially. Each new generation is
* assigned 1 + the current highest generation number.
*/
typedef uint64_t GenerationNumber;
/**
* A generation is a revision of a profile.
*
* Each generation is a mapping (key-value pair) from an identifier
* (`number`) to a store object (specified by `path`).
*/
struct Generation
{
/**
* The number of a generation is its unique identifier within the
* profile.
*/
GenerationNumber number;
/**
* The store path identifies the store object that is the contents
* of the generation.
*
* These store paths / objects are not unique to the generation
* within a profile. Nix tries to ensure successive generations have
* distinct contents to avoid bloat, but nothing stops two
* non-adjacent generations from having the same contents.
*
* @todo Use `StorePath` instead of `Path`?
*/
Path path;
/**
* When the generation was created. This is extra metadata about the
* generation used to make garbage collecting old generations more
* convenient.
*/
time_t creationTime;
};
/**
* All the generations of a profile
*/
typedef std::list<Generation> Generations;
/**
* Returns the list of currently present generations for the specified
* profile, sorted by generation number. Also returns the number of
* the current generation.
* Find all generations for the given profile.
*
* @param profile A profile specified by its name and location combined
* into a path. E.g. if "foo" is the name of the profile, and "/bar/baz"
* is the directory it is in, then the path "/bar/baz/foo" would be the
* argument for this parameter.
*
* @return The pair of:
*
* - The list of currently present generations for the specified profile,
* sorted by ascending generation number.
*
* - The number of the current/active generation.
*
* Note that the current/active generation need not be the latest one.
*/
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore;
Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
/**
* Create a new generation of the given profile
*
* If the previous generation (not the currently active one!) has a
* distinct store object, a fresh generation number is mapped to the
* given store object, referenced by path. Otherwise, the previous
* generation is assumed.
*
* The behavior of reusing existing generations like this makes this
* procedure idempotent. It also avoids clutter.
*/
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath);
/**
* Unconditionally delete a generation
*
* @param profile A profile specified by its name and location combined into a path.
*
* @param gen The generation number specifying exactly which generation
* to delete.
*
* Because there is no check of whether the generation to delete is
* active, this is somewhat unsafe.
*
* @todo Should we expose this at all?
*/
void deleteGeneration(const Path & profile, GenerationNumber gen);
/**
* Delete the given set of generations.
*
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
*
* @param gensToDelete The generations to delete, specified by a set of
* numbers.
*
* @param dryRun Log what would be deleted instead of actually doing
* so.
*
* Trying to delete the currently active generation will fail, and cause
* no generations to be deleted.
*/
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
/**
* Delete generations older than `max` passed the current generation.
*
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
*
* @param max How many generations to keep up to the current one. Must
* be at least 1 so we don't delete the current one.
*
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
/**
* Delete all generations other than the current one
*
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
*
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteOldGenerations(const Path & profile, bool dryRun);
/**
* Delete generations older than `t`, except for the most recent one
* older than `t`.
*
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
*
* @param dryRun Log what would be deleted instead of actually doing
* so.
*/
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun);
/**
* Parse a temp spec intended for `deleteGenerationsOlderThan()`.
*
* Throws an exception if `timeSpec` fails to parse.
*/
time_t parseOlderThanTimeSpec(std::string_view timeSpec);
/**
* Smaller wrapper around `replaceSymlink` for replacing the current
* generation of a profile. Does not enforce proper structure.
*
* @todo Always use `switchGeneration()` instead, and delete this.
*/
void switchLink(Path link, Path target);
/**