mirror of
https://github.com/NixOS/nix.git
synced 2025-11-20 17:29:36 +01:00
Merge branch 'read-only-local-store' into overlayfs-store
This commit is contained in:
Ben Radford
commit
71f3bad749
79 changed files with 1152 additions and 367 deletions
|
|
@ -4,7 +4,7 @@
|
|||
#include "worker.hh"
|
||||
#include "builtins.hh"
|
||||
#include "builtins/buildenv.hh"
|
||||
#include "references.hh"
|
||||
#include "path-references.hh"
|
||||
#include "finally.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -2389,18 +2389,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
continue;
|
||||
auto references = *referencesOpt;
|
||||
|
||||
auto rewriteOutput = [&]() {
|
||||
auto rewriteOutput = [&](const StringMap & rewrites) {
|
||||
/* Apply hash rewriting if necessary. */
|
||||
if (!outputRewrites.empty()) {
|
||||
if (!rewrites.empty()) {
|
||||
debug("rewriting hashes in '%1%'; cross fingers", actualPath);
|
||||
|
||||
/* FIXME: this is in-memory. */
|
||||
StringSink sink;
|
||||
dumpPath(actualPath, sink);
|
||||
/* FIXME: Is this actually streaming? */
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink(rewrites, nextSink);
|
||||
dumpPath(actualPath, rsink);
|
||||
rsink.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
restorePath(tmpPath, *source);
|
||||
deletePath(actualPath);
|
||||
sink.s = rewriteStrings(sink.s, outputRewrites);
|
||||
StringSource source(sink.s);
|
||||
restorePath(actualPath, source);
|
||||
movePath(tmpPath, actualPath);
|
||||
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
we don't have to do another traversal. */
|
||||
|
|
@ -2449,7 +2452,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
"since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
|
||||
actualPath);
|
||||
}
|
||||
rewriteOutput();
|
||||
rewriteOutput(outputRewrites);
|
||||
/* FIXME optimize and deduplicate with addToStore */
|
||||
std::string oldHashPart { scratchPath->hashPart() };
|
||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||
|
|
@ -2487,16 +2490,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
Hash::dummy,
|
||||
};
|
||||
if (*scratchPath != newInfo0.path) {
|
||||
// Also rewrite the output path
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
|
||||
dumpPath(actualPath, rsink2);
|
||||
rsink2.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
restorePath(tmpPath, *source);
|
||||
deletePath(actualPath);
|
||||
movePath(tmpPath, actualPath);
|
||||
// If the path has some self-references, we need to rewrite
|
||||
// them.
|
||||
// (note that this doesn't invalidate the ca hash we calculated
|
||||
// above because it's computed *modulo the self-references*, so
|
||||
// it already takes this rewrite into account).
|
||||
rewriteOutput(
|
||||
StringMap{{oldHashPart,
|
||||
std::string(newInfo0.path.hashPart())}});
|
||||
}
|
||||
|
||||
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
|
||||
|
|
@ -2518,7 +2519,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
outputRewrites.insert_or_assign(
|
||||
std::string { scratchPath->hashPart() },
|
||||
std::string { requiredFinalPath.hashPart() });
|
||||
rewriteOutput();
|
||||
rewriteOutput(outputRewrites);
|
||||
auto narHashAndSize = hashPath(htSHA256, actualPath);
|
||||
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
|
||||
newInfo0.narSize = narHashAndSize.second;
|
||||
|
|
|
|||
|
|
@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer
|
|||
size_t realSize = size * nmemb;
|
||||
std::string line((char *) contents, realSize);
|
||||
printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
|
||||
|
||||
static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
|
||||
std::smatch match;
|
||||
if (std::regex_match(line, match, statusLine)) {
|
||||
if (std::smatch match; std::regex_match(line, match, statusLine)) {
|
||||
result.etag = "";
|
||||
result.data.clear();
|
||||
result.bodySize = 0;
|
||||
|
|
@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer
|
|||
acceptRanges = false;
|
||||
encoding = "";
|
||||
} else {
|
||||
|
||||
auto i = line.find(':');
|
||||
if (i != std::string::npos) {
|
||||
std::string name = toLower(trim(line.substr(0, i)));
|
||||
|
||||
if (name == "etag") {
|
||||
result.etag = trim(line.substr(i + 1));
|
||||
/* Hack to work around a GitHub bug: it sends
|
||||
|
|
@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer
|
|||
debug("shutting down on 200 HTTP response with expected ETag");
|
||||
return 0;
|
||||
}
|
||||
} else if (name == "content-encoding")
|
||||
}
|
||||
|
||||
else if (name == "content-encoding")
|
||||
encoding = trim(line.substr(i + 1));
|
||||
|
||||
else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
|
||||
acceptRanges = true;
|
||||
|
||||
else if (name == "link" || name == "x-amz-meta-link") {
|
||||
auto value = trim(line.substr(i + 1));
|
||||
static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase);
|
||||
if (std::smatch match; std::regex_match(value, match, linkRegex))
|
||||
result.immutableUrl = match.str(1);
|
||||
else
|
||||
debug("got invalid link header '%s'", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return realSize;
|
||||
|
|
@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer
|
|||
{
|
||||
auto httpStatus = getHTTPStatus();
|
||||
|
||||
char * effectiveUriCStr;
|
||||
char * effectiveUriCStr = nullptr;
|
||||
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
|
||||
if (effectiveUriCStr)
|
||||
result.effectiveUri = effectiveUriCStr;
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ struct FileTransferResult
|
|||
std::string effectiveUri;
|
||||
std::string data;
|
||||
uint64_t bodySize = 0;
|
||||
/* An "immutable" URL for this resource (i.e. one whose contents
|
||||
will never change), as returned by the `Link: <url>;
|
||||
rel="immutable"` header. */
|
||||
std::optional<std::string> immutableUrl;
|
||||
};
|
||||
|
||||
class Store;
|
||||
|
|
|
|||
|
|
@ -77,7 +77,30 @@ Settings::Settings()
|
|||
allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
|
||||
#endif
|
||||
|
||||
buildHook = getSelfExe().value_or("nix") + " __build-remote";
|
||||
/* Set the build hook location
|
||||
|
||||
For builds we perform a self-invocation, so Nix has to be self-aware.
|
||||
That is, it has to know where it is installed. We don't think it's sentient.
|
||||
|
||||
Normally, nix is installed according to `nixBinDir`, which is set at compile time,
|
||||
but can be overridden. This makes for a great default that works even if this
|
||||
code is linked as a library into some other program whose main is not aware
|
||||
that it might need to be a build remote hook.
|
||||
|
||||
However, it may not have been installed at all. For example, if it's a static build,
|
||||
there's a good chance that it has been moved out of its installation directory.
|
||||
That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the
|
||||
current executable, using `getSelfExe()`.
|
||||
|
||||
As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible.
|
||||
If you're porting Nix to a new platform, that might be good enough for a while, but
|
||||
you'll want to improve `getSelfExe()` to work on your platform.
|
||||
*/
|
||||
std::string nixExePath = nixBinDir + "/nix";
|
||||
if (!pathExists(nixExePath)) {
|
||||
nixExePath = getSelfExe().value_or("nix");
|
||||
}
|
||||
buildHook = nixExePath + " __build-remote";
|
||||
}
|
||||
|
||||
void loadConfFile()
|
||||
|
|
|
|||
|
|
@ -710,20 +710,19 @@ public:
|
|||
Strings{"https://cache.nixos.org/"},
|
||||
"substituters",
|
||||
R"(
|
||||
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
|
||||
to be used as substituters, separated by whitespace.
|
||||
Substituters are tried based on their Priority value, which each substituter can set
|
||||
independently. Lower value means higher priority.
|
||||
The default is `https://cache.nixos.org`, with a Priority of 40.
|
||||
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
|
||||
A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
|
||||
|
||||
At least one of the following conditions must be met for Nix to use
|
||||
a substituter:
|
||||
Substituters are tried based on their priority value, which each substituter can set independently.
|
||||
Lower value means higher priority.
|
||||
The default is `https://cache.nixos.org`, which has a priority of 40.
|
||||
|
||||
At least one of the following conditions must be met for Nix to use a substituter:
|
||||
|
||||
- the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
|
||||
- the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
|
||||
|
||||
In addition, each store path should be trusted as described
|
||||
in [`trusted-public-keys`](#conf-trusted-public-keys)
|
||||
In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
|
||||
)",
|
||||
{"binary-caches"}};
|
||||
|
||||
|
|
|
|||
|
|
@ -52,14 +52,15 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
|
|||
R"(
|
||||
Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
|
||||
|
||||
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed).
|
||||
This causes it to fail if the database is on a read-only filesystem.
|
||||
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
|
||||
|
||||
Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
|
||||
|
||||
**Warning**
|
||||
Do not use this unless the filesystem is read-only.
|
||||
Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
|
||||
> **Warning**
|
||||
> Do not use this unless the filesystem is read-only.
|
||||
>
|
||||
> Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
|
||||
> While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
|
||||
)"};
|
||||
|
||||
const std::string name() override { return "Local Store"; }
|
||||
|
|
|
|||
73
src/libstore/path-references.cc
Normal file
73
src/libstore/path-references.cc
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
#include "path-references.hh"
|
||||
#include "hash.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <algorithm>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
|
||||
: RefScanSink(std::move(hashes))
|
||||
, backMap(std::move(backMap))
|
||||
{ }
|
||||
|
||||
PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
|
||||
{
|
||||
StringSet hashes;
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
for (auto & i : refs) {
|
||||
std::string hashPart(i.hashPart());
|
||||
auto inserted = backMap.emplace(hashPart, i).second;
|
||||
assert(inserted);
|
||||
hashes.insert(hashPart);
|
||||
}
|
||||
|
||||
return PathRefScanSink(std::move(hashes), std::move(backMap));
|
||||
}
|
||||
|
||||
StorePathSet PathRefScanSink::getResultPaths()
|
||||
{
|
||||
/* Map the hashes found back to their store paths. */
|
||||
StorePathSet found;
|
||||
for (auto & i : getResult()) {
|
||||
auto j = backMap.find(i);
|
||||
assert(j != backMap.end());
|
||||
found.insert(j->second);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(
|
||||
const std::string & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
HashSink hashSink { htSHA256 };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<StorePathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
StorePathSet scanForReferences(
|
||||
Sink & toTee,
|
||||
const Path & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
|
||||
TeeSink sink { refsSink, toTee };
|
||||
|
||||
/* Look for the hashes in the NAR dump of the path. */
|
||||
dumpPath(path, sink);
|
||||
|
||||
return refsSink.getResultPaths();
|
||||
}
|
||||
|
||||
}
|
||||
25
src/libstore/path-references.hh
Normal file
25
src/libstore/path-references.hh
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#pragma once
|
||||
|
||||
#include "references.hh"
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
|
||||
|
||||
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
|
||||
|
||||
class PathRefScanSink : public RefScanSink
|
||||
{
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
|
||||
|
||||
public:
|
||||
|
||||
static PathRefScanSink fromPaths(const StorePathSet & refs);
|
||||
|
||||
StorePathSet getResultPaths();
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -13,8 +13,10 @@
|
|||
namespace nix {
|
||||
|
||||
|
||||
/* Parse a generation name of the format
|
||||
`<profilename>-<number>-link'. */
|
||||
/**
|
||||
* Parse a generation name of the format
|
||||
* `<profilename>-<number>-link'.
|
||||
*/
|
||||
static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name)
|
||||
{
|
||||
if (name.substr(0, profileName.size() + 1) != profileName + "-") return {};
|
||||
|
|
@ -28,7 +30,6 @@ static std::optional<GenerationNumber> parseName(const std::string & profileName
|
|||
}
|
||||
|
||||
|
||||
|
||||
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
|
||||
{
|
||||
Generations gens;
|
||||
|
|
@ -61,15 +62,16 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
|
|||
}
|
||||
|
||||
|
||||
static void makeName(const Path & profile, GenerationNumber num,
|
||||
Path & outLink)
|
||||
/**
|
||||
* Create a generation name that can be parsed by `parseName()`.
|
||||
*/
|
||||
static Path makeName(const Path & profile, GenerationNumber num)
|
||||
{
|
||||
Path prefix = fmt("%1%-%2%", profile, num);
|
||||
outLink = prefix + "-link";
|
||||
return fmt("%s-%s-link", profile, num);
|
||||
}
|
||||
|
||||
|
||||
Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
|
||||
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath)
|
||||
{
|
||||
/* The new generation number should be higher than old the
|
||||
previous ones. */
|
||||
|
|
@ -79,7 +81,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
|
|||
if (gens.size() > 0) {
|
||||
Generation last = gens.back();
|
||||
|
||||
if (readLink(last.path) == store->printStorePath(outPath)) {
|
||||
if (readLink(last.path) == store.printStorePath(outPath)) {
|
||||
/* We only create a new generation symlink if it differs
|
||||
from the last one.
|
||||
|
||||
|
|
@ -89,7 +91,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
|
|||
return last.path;
|
||||
}
|
||||
|
||||
num = gens.back().number;
|
||||
num = last.number;
|
||||
} else {
|
||||
num = 0;
|
||||
}
|
||||
|
|
@ -100,9 +102,8 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath)
|
|||
to the permanent roots (of which the GC would have a stale
|
||||
view). If we didn't do it this way, the GC might remove the
|
||||
user environment etc. we've just built. */
|
||||
Path generation;
|
||||
makeName(profile, num + 1, generation);
|
||||
store->addPermRoot(outPath, generation);
|
||||
Path generation = makeName(profile, num + 1);
|
||||
store.addPermRoot(outPath, generation);
|
||||
|
||||
return generation;
|
||||
}
|
||||
|
|
@ -117,12 +118,19 @@ static void removeFile(const Path & path)
|
|||
|
||||
void deleteGeneration(const Path & profile, GenerationNumber gen)
|
||||
{
|
||||
Path generation;
|
||||
makeName(profile, gen, generation);
|
||||
Path generation = makeName(profile, gen);
|
||||
removeFile(generation);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Delete a generation with dry-run mode.
|
||||
*
|
||||
* Like `deleteGeneration()` but:
|
||||
*
|
||||
* - We log what we are going to do.
|
||||
*
|
||||
* - We only actually delete if `dryRun` is false.
|
||||
*/
|
||||
static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
|
||||
{
|
||||
if (dryRun)
|
||||
|
|
@ -150,27 +158,36 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Advanced the iterator until the given predicate `cond` returns `true`.
|
||||
*/
|
||||
static inline void iterDropUntil(Generations & gens, auto && i, auto && cond)
|
||||
{
|
||||
for (; i != gens.rend() && !cond(*i); ++i);
|
||||
}
|
||||
|
||||
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
|
||||
{
|
||||
if (max == 0)
|
||||
throw Error("Must keep at least one generation, otherwise the current one would be deleted");
|
||||
|
||||
PathLocks lock;
|
||||
lockProfile(lock, profile);
|
||||
|
||||
bool fromCurGen = false;
|
||||
auto [gens, curGen] = findGenerations(profile);
|
||||
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
|
||||
if (i->number == curGen) {
|
||||
fromCurGen = true;
|
||||
max--;
|
||||
continue;
|
||||
}
|
||||
if (fromCurGen) {
|
||||
if (max) {
|
||||
max--;
|
||||
continue;
|
||||
}
|
||||
deleteGeneration2(profile, i->number, dryRun);
|
||||
}
|
||||
}
|
||||
auto [gens, _curGen] = findGenerations(profile);
|
||||
auto curGen = _curGen;
|
||||
|
||||
auto i = gens.rbegin();
|
||||
|
||||
// Find the current generation
|
||||
iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; });
|
||||
|
||||
// Skip over `max` generations, preserving them
|
||||
for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep);
|
||||
|
||||
// Delete the rest
|
||||
for (; i != gens.rend(); ++i)
|
||||
deleteGeneration2(profile, i->number, dryRun);
|
||||
}
|
||||
|
||||
void deleteOldGenerations(const Path & profile, bool dryRun)
|
||||
|
|
@ -193,23 +210,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
|
|||
|
||||
auto [gens, curGen] = findGenerations(profile);
|
||||
|
||||
bool canDelete = false;
|
||||
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
|
||||
if (canDelete) {
|
||||
assert(i->creationTime < t);
|
||||
if (i->number != curGen)
|
||||
deleteGeneration2(profile, i->number, dryRun);
|
||||
} else if (i->creationTime < t) {
|
||||
/* We may now start deleting generations, but we don't
|
||||
delete this generation yet, because this generation was
|
||||
still the one that was active at the requested point in
|
||||
time. */
|
||||
canDelete = true;
|
||||
}
|
||||
auto i = gens.rbegin();
|
||||
|
||||
// Predicate that the generation is older than the given time.
|
||||
auto older = [&](auto & g) { return g.creationTime < t; };
|
||||
|
||||
// Find the first older generation, if one exists
|
||||
iterDropUntil(gens, i, older);
|
||||
|
||||
/* Take the previous generation
|
||||
|
||||
We don't want delete this one yet because it
|
||||
existed at the requested point in time, and
|
||||
we want to be able to roll back to it. */
|
||||
if (i != gens.rend()) ++i;
|
||||
|
||||
// Delete all previous generations (unless current).
|
||||
for (; i != gens.rend(); ++i) {
|
||||
/* Creating date and generations should be monotonic, so lower
|
||||
numbered derivations should also be older. */
|
||||
assert(older(*i));
|
||||
if (i->number != curGen)
|
||||
deleteGeneration2(profile, i->number, dryRun);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun)
|
||||
time_t parseOlderThanTimeSpec(std::string_view timeSpec)
|
||||
{
|
||||
if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
|
||||
throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
|
||||
|
|
@ -221,9 +248,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec,
|
|||
if (!days || *days < 1)
|
||||
throw UsageError("invalid number of days specifier '%1%'", timeSpec);
|
||||
|
||||
time_t oldTime = curTime - *days * 24 * 3600;
|
||||
|
||||
deleteGenerationsOlderThan(profile, oldTime, dryRun);
|
||||
return curTime - *days * 24 * 3600;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
#pragma once
|
||||
///@file
|
||||
/**
|
||||
* @file Implementation of Profiles.
|
||||
*
|
||||
* See the manual for additional information.
|
||||
*/
|
||||
|
||||
#include "types.hh"
|
||||
#include "types.hh"
|
||||
#include "pathlocks.hh"
|
||||
|
||||
#include <time.h>
|
||||
|
|
@ -12,41 +16,166 @@ namespace nix {
|
|||
class StorePath;
|
||||
|
||||
|
||||
/**
|
||||
* A positive number identifying a generation for a given profile.
|
||||
*
|
||||
* Generation numbers are assigned sequentially. Each new generation is
|
||||
* assigned 1 + the current highest generation number.
|
||||
*/
|
||||
typedef uint64_t GenerationNumber;
|
||||
|
||||
/**
|
||||
* A generation is a revision of a profile.
|
||||
*
|
||||
* Each generation is a mapping (key-value pair) from an identifier
|
||||
* (`number`) to a store object (specified by `path`).
|
||||
*/
|
||||
struct Generation
|
||||
{
|
||||
/**
|
||||
* The number of a generation is its unique identifier within the
|
||||
* profile.
|
||||
*/
|
||||
GenerationNumber number;
|
||||
/**
|
||||
* The store path identifies the store object that is the contents
|
||||
* of the generation.
|
||||
*
|
||||
* These store paths / objects are not unique to the generation
|
||||
* within a profile. Nix tries to ensure successive generations have
|
||||
* distinct contents to avoid bloat, but nothing stops two
|
||||
* non-adjacent generations from having the same contents.
|
||||
*
|
||||
* @todo Use `StorePath` instead of `Path`?
|
||||
*/
|
||||
Path path;
|
||||
|
||||
/**
|
||||
* When the generation was created. This is extra metadata about the
|
||||
* generation used to make garbage collecting old generations more
|
||||
* convenient.
|
||||
*/
|
||||
time_t creationTime;
|
||||
};
|
||||
|
||||
/**
|
||||
* All the generations of a profile
|
||||
*/
|
||||
typedef std::list<Generation> Generations;
|
||||
|
||||
|
||||
/**
|
||||
* Returns the list of currently present generations for the specified
|
||||
* profile, sorted by generation number. Also returns the number of
|
||||
* the current generation.
|
||||
* Find all generations for the given profile.
|
||||
*
|
||||
* @param profile A profile specified by its name and location combined
|
||||
* into a path. E.g. if "foo" is the name of the profile, and "/bar/baz"
|
||||
* is the directory it is in, then the path "/bar/baz/foo" would be the
|
||||
* argument for this parameter.
|
||||
*
|
||||
* @return The pair of:
|
||||
*
|
||||
* - The list of currently present generations for the specified profile,
|
||||
* sorted by ascending generation number.
|
||||
*
|
||||
* - The number of the current/active generation.
|
||||
*
|
||||
* Note that the current/active generation need not be the latest one.
|
||||
*/
|
||||
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
|
||||
|
||||
class LocalFSStore;
|
||||
|
||||
Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath);
|
||||
/**
|
||||
* Create a new generation of the given profile
|
||||
*
|
||||
* If the previous generation (not the currently active one!) has a
|
||||
* distinct store object, a fresh generation number is mapped to the
|
||||
* given store object, referenced by path. Otherwise, the previous
|
||||
* generation is assumed.
|
||||
*
|
||||
* The behavior of reusing existing generations like this makes this
|
||||
* procedure idempotent. It also avoids clutter.
|
||||
*/
|
||||
Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath);
|
||||
|
||||
/**
|
||||
* Unconditionally delete a generation
|
||||
*
|
||||
* @param profile A profile specified by its name and location combined into a path.
|
||||
*
|
||||
* @param gen The generation number specifying exactly which generation
|
||||
* to delete.
|
||||
*
|
||||
* Because there is no check of whether the generation to delete is
|
||||
* active, this is somewhat unsafe.
|
||||
*
|
||||
* @todo Should we expose this at all?
|
||||
*/
|
||||
void deleteGeneration(const Path & profile, GenerationNumber gen);
|
||||
|
||||
/**
|
||||
* Delete the given set of generations.
|
||||
*
|
||||
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
|
||||
*
|
||||
* @param gensToDelete The generations to delete, specified by a set of
|
||||
* numbers.
|
||||
*
|
||||
* @param dryRun Log what would be deleted instead of actually doing
|
||||
* so.
|
||||
*
|
||||
* Trying to delete the currently active generation will fail, and cause
|
||||
* no generations to be deleted.
|
||||
*/
|
||||
void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
|
||||
|
||||
/**
|
||||
* Delete generations older than `max` passed the current generation.
|
||||
*
|
||||
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
|
||||
*
|
||||
* @param max How many generations to keep up to the current one. Must
|
||||
* be at least 1 so we don't delete the current one.
|
||||
*
|
||||
* @param dryRun Log what would be deleted instead of actually doing
|
||||
* so.
|
||||
*/
|
||||
void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
|
||||
|
||||
/**
|
||||
* Delete all generations other than the current one
|
||||
*
|
||||
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
|
||||
*
|
||||
* @param dryRun Log what would be deleted instead of actually doing
|
||||
* so.
|
||||
*/
|
||||
void deleteOldGenerations(const Path & profile, bool dryRun);
|
||||
|
||||
/**
|
||||
* Delete generations older than `t`, except for the most recent one
|
||||
* older than `t`.
|
||||
*
|
||||
* @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete.
|
||||
*
|
||||
* @param dryRun Log what would be deleted instead of actually doing
|
||||
* so.
|
||||
*/
|
||||
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
|
||||
|
||||
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun);
|
||||
/**
|
||||
* Parse a temp spec intended for `deleteGenerationsOlderThan()`.
|
||||
*
|
||||
* Throws an exception if `timeSpec` fails to parse.
|
||||
*/
|
||||
time_t parseOlderThanTimeSpec(std::string_view timeSpec);
|
||||
|
||||
/**
|
||||
* Smaller wrapper around `replaceSymlink` for replacing the current
|
||||
* generation of a profile. Does not enforce proper structure.
|
||||
*
|
||||
* @todo Always use `switchGeneration()` instead, and delete this.
|
||||
*/
|
||||
void switchLink(Path link, Path target);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,188 +0,0 @@
|
|||
#include "references.hh"
|
||||
#include "hash.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
static size_t refLength = 32; /* characters */
|
||||
|
||||
|
||||
static void search(
|
||||
std::string_view s,
|
||||
StringSet & hashes,
|
||||
StringSet & seen)
|
||||
{
|
||||
static std::once_flag initialised;
|
||||
static bool isBase32[256];
|
||||
std::call_once(initialised, [](){
|
||||
for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
|
||||
for (unsigned int i = 0; i < base32Chars.size(); ++i)
|
||||
isBase32[(unsigned char) base32Chars[i]] = true;
|
||||
});
|
||||
|
||||
for (size_t i = 0; i + refLength <= s.size(); ) {
|
||||
int j;
|
||||
bool match = true;
|
||||
for (j = refLength - 1; j >= 0; --j)
|
||||
if (!isBase32[(unsigned char) s[i + j]]) {
|
||||
i += j + 1;
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
if (!match) continue;
|
||||
std::string ref(s.substr(i, refLength));
|
||||
if (hashes.erase(ref)) {
|
||||
debug("found reference to '%1%' at offset '%2%'", ref, i);
|
||||
seen.insert(ref);
|
||||
}
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void RefScanSink::operator () (std::string_view data)
|
||||
{
|
||||
/* It's possible that a reference spans the previous and current
|
||||
fragment, so search in the concatenation of the tail of the
|
||||
previous fragment and the start of the current fragment. */
|
||||
auto s = tail;
|
||||
auto tailLen = std::min(data.size(), refLength);
|
||||
s.append(data.data(), tailLen);
|
||||
search(s, hashes, seen);
|
||||
|
||||
search(data, hashes, seen);
|
||||
|
||||
auto rest = refLength - tailLen;
|
||||
if (rest < tail.size())
|
||||
tail = tail.substr(tail.size() - rest);
|
||||
tail.append(data.data() + data.size() - tailLen, tailLen);
|
||||
}
|
||||
|
||||
|
||||
PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
|
||||
: RefScanSink(std::move(hashes))
|
||||
, backMap(std::move(backMap))
|
||||
{ }
|
||||
|
||||
PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
|
||||
{
|
||||
StringSet hashes;
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
for (auto & i : refs) {
|
||||
std::string hashPart(i.hashPart());
|
||||
auto inserted = backMap.emplace(hashPart, i).second;
|
||||
assert(inserted);
|
||||
hashes.insert(hashPart);
|
||||
}
|
||||
|
||||
return PathRefScanSink(std::move(hashes), std::move(backMap));
|
||||
}
|
||||
|
||||
StorePathSet PathRefScanSink::getResultPaths()
|
||||
{
|
||||
/* Map the hashes found back to their store paths. */
|
||||
StorePathSet found;
|
||||
for (auto & i : getResult()) {
|
||||
auto j = backMap.find(i);
|
||||
assert(j != backMap.end());
|
||||
found.insert(j->second);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(
|
||||
const std::string & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
HashSink hashSink { htSHA256 };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<StorePathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
StorePathSet scanForReferences(
|
||||
Sink & toTee,
|
||||
const Path & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
|
||||
TeeSink sink { refsSink, toTee };
|
||||
|
||||
/* Look for the hashes in the NAR dump of the path. */
|
||||
dumpPath(path, sink);
|
||||
|
||||
return refsSink.getResultPaths();
|
||||
}
|
||||
|
||||
|
||||
RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
|
||||
: from(from), to(to), nextSink(nextSink)
|
||||
{
|
||||
assert(from.size() == to.size());
|
||||
}
|
||||
|
||||
void RewritingSink::operator () (std::string_view data)
|
||||
{
|
||||
std::string s(prev);
|
||||
s.append(data);
|
||||
|
||||
size_t j = 0;
|
||||
while ((j = s.find(from, j)) != std::string::npos) {
|
||||
matches.push_back(pos + j);
|
||||
s.replace(j, from.size(), to);
|
||||
}
|
||||
|
||||
prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
|
||||
|
||||
auto consumed = s.size() - prev.size();
|
||||
|
||||
pos += consumed;
|
||||
|
||||
if (consumed) nextSink(s.substr(0, consumed));
|
||||
}
|
||||
|
||||
void RewritingSink::flush()
|
||||
{
|
||||
if (prev.empty()) return;
|
||||
pos += prev.size();
|
||||
nextSink(prev);
|
||||
prev.clear();
|
||||
}
|
||||
|
||||
HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
|
||||
: hashSink(ht)
|
||||
, rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
|
||||
{
|
||||
}
|
||||
|
||||
void HashModuloSink::operator () (std::string_view data)
|
||||
{
|
||||
rewritingSink(data);
|
||||
}
|
||||
|
||||
HashResult HashModuloSink::finish()
|
||||
{
|
||||
rewritingSink.flush();
|
||||
|
||||
/* Hash the positions of the self-references. This ensures that a
|
||||
NAR with self-references and a NAR with some of the
|
||||
self-references already zeroed out do not produce a hash
|
||||
collision. FIXME: proof. */
|
||||
for (auto & pos : rewritingSink.matches)
|
||||
hashSink(fmt("|%d", pos));
|
||||
|
||||
auto h = hashSink.finish();
|
||||
return {h.first, rewritingSink.pos};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "hash.hh"
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
|
||||
|
||||
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
|
||||
|
||||
class RefScanSink : public Sink
|
||||
{
|
||||
StringSet hashes;
|
||||
StringSet seen;
|
||||
|
||||
std::string tail;
|
||||
|
||||
public:
|
||||
|
||||
RefScanSink(StringSet && hashes) : hashes(hashes)
|
||||
{ }
|
||||
|
||||
StringSet & getResult()
|
||||
{ return seen; }
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
};
|
||||
|
||||
class PathRefScanSink : public RefScanSink
|
||||
{
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
|
||||
|
||||
public:
|
||||
|
||||
static PathRefScanSink fromPaths(const StorePathSet & refs);
|
||||
|
||||
StorePathSet getResultPaths();
|
||||
};
|
||||
|
||||
struct RewritingSink : Sink
|
||||
{
|
||||
std::string from, to, prev;
|
||||
Sink & nextSink;
|
||||
uint64_t pos = 0;
|
||||
|
||||
std::vector<uint64_t> matches;
|
||||
|
||||
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
|
||||
void flush();
|
||||
};
|
||||
|
||||
struct HashModuloSink : AbstractHashSink
|
||||
{
|
||||
HashSink hashSink;
|
||||
RewritingSink rewritingSink;
|
||||
|
||||
HashModuloSink(HashType ht, const std::string & modulus);
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
|
||||
HashResult finish() override;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -29,7 +29,7 @@ enum class SQLiteOpenMode {
|
|||
* Use this mode if the database is on a read-only filesystem.
|
||||
* Fails with an error if the database does not exist.
|
||||
*/
|
||||
Immutable
|
||||
Immutable,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue