mirror of
https://github.com/NixOS/nix.git
synced 2025-11-26 04:00:59 +01:00
Merged to R9439. Fixed a computeFSClosure bug. The state garbage colletor basically works, Missing items: State locks, shared state and Topological sort
This commit is contained in:
Wouter den Breejen
parent
7d82fd16e9
commit
16410fc714
12 changed files with 372 additions and 154 deletions
|
|
@ -1,5 +1,7 @@
|
|||
#! /bin/sh -e
|
||||
|
||||
make clean
|
||||
|
||||
export nixstatepath=/nixstate2/nix
|
||||
export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal
|
||||
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
#! /usr/bin/perl -w
|
||||
|
||||
use strict;
|
||||
use File::Basename;
|
||||
|
||||
|
||||
my @paths = ("/nix/store");
|
||||
|
||||
|
||||
print "hashing...\n";
|
||||
|
||||
my $hashList = "/tmp/nix-optimise-hash-list";
|
||||
|
||||
system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0
|
||||
or die "cannot hash store files";
|
||||
|
||||
|
||||
print "sorting by hash...\n";
|
||||
|
||||
system("sort $hashList > $hashList.sorted") == 0
|
||||
or die "cannot sort list";
|
||||
|
||||
|
||||
sub atomicLink {
|
||||
my $target = shift;
|
||||
my $new = shift;
|
||||
my $tmpNew = "${new}_optimise.$$";
|
||||
|
||||
# Make the directory writable temporarily.
|
||||
my $dir = dirname $new;
|
||||
my @st = stat $dir or die;
|
||||
|
||||
chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!";
|
||||
|
||||
link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!";
|
||||
|
||||
rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!";
|
||||
|
||||
chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!";
|
||||
utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!";
|
||||
}
|
||||
|
||||
|
||||
print "hard-linking...\n";
|
||||
|
||||
open LIST, "<$hashList.sorted" or die;
|
||||
|
||||
my $prevFile;
|
||||
my $prevHash;
|
||||
my $prevInode;
|
||||
my $prevExec;
|
||||
|
||||
my $totalSpace = 0;
|
||||
my $savedSpace = 0;
|
||||
|
||||
while (<LIST>) {
|
||||
/^([0-9a-f]*)\s+(.*)$/ or die;
|
||||
my $curFile = $2;
|
||||
my $curHash = $1;
|
||||
|
||||
my @st = stat $curFile or die;
|
||||
next if ($st[2] & 0222) != 0; # skip writable files
|
||||
|
||||
my $fileSize = $st[7];
|
||||
$totalSpace += $fileSize;
|
||||
my $isExec = ($st[2] & 0111) == 0111;
|
||||
|
||||
if (defined $prevHash && $curHash eq $prevHash
|
||||
&& $prevExec == $isExec)
|
||||
{
|
||||
|
||||
if ($st[1] != $prevInode) {
|
||||
print "$curFile = $prevFile\n";
|
||||
atomicLink $prevFile, $curFile;
|
||||
$savedSpace += $fileSize;
|
||||
}
|
||||
|
||||
} else {
|
||||
$prevFile = $curFile;
|
||||
$prevHash = $curHash;
|
||||
$prevInode = $st[1];
|
||||
$prevExec = ($st[2] & 0111) == 0111;
|
||||
}
|
||||
}
|
||||
|
||||
print "total space = $totalSpace\n";
|
||||
print "saved space = $savedSpace\n";
|
||||
my $savings = ($savedSpace / $totalSpace) * 100.0;
|
||||
print "savings = $savings %\n";
|
||||
|
||||
close LIST;
|
||||
|
|
@ -452,11 +452,11 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
/* Acquire the global GC root. This prevents
|
||||
a) New roots from being added.
|
||||
b) Processes from creating new temporary root files. */
|
||||
AutoCloseFD fdGCLock = openGCLock(ltWrite);
|
||||
AutoCloseFD fdGCLock = openGCLock(ltWrite); //TODO !!!!!!!!!!!!!!!!!!! GET STATE LOCKS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
/* Find the roots. Since we've grabbed the GC lock, the set of
|
||||
permanent roots cannot increase now. */
|
||||
Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true); //TODO Also find state roots --> nah ? TODO include sharing of state.
|
||||
Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
|
||||
|
||||
PathSet roots;
|
||||
for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
|
||||
|
|
@ -480,9 +480,10 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
PathSet livePaths;
|
||||
for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){
|
||||
printMsg(lvlError, format("CHECK '%1%'") % *i);
|
||||
computeFSClosure(canonPath(*i), livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
|
||||
computeFSClosure(canonPath(*i), livePaths, true, true, 0);
|
||||
}
|
||||
|
||||
|
||||
printMsg(lvlError, format("STAGE X"));
|
||||
|
||||
if (gcKeepDerivations) {
|
||||
for (PathSet::iterator i = livePaths.begin();
|
||||
|
|
@ -496,28 +497,29 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j)
|
||||
// We send each drv to computeFSClosure
|
||||
if (*j != "" && store->isValidPath(*j))
|
||||
computeFSClosure(*j, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
|
||||
computeFSClosure(*j, livePaths, true, true, 0);
|
||||
}
|
||||
else if (store->isValidPath(*i)){
|
||||
printMsg(lvlError, format("CHECK-STORE '%1%'") % *i);
|
||||
Path deriver = store->queryDeriver(*i);
|
||||
printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver);
|
||||
//printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver);
|
||||
|
||||
/* Note that the deriver need not be valid (e.g., if we
|
||||
previously ran the collector with `gcKeepDerivations'
|
||||
turned off). */
|
||||
if (deriver != "" && store->isValidPath(deriver))
|
||||
computeFSClosure(deriver, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE???
|
||||
computeFSClosure(deriver, livePaths, true, true, 0);
|
||||
}
|
||||
|
||||
//else if (store->isValidStatePath(*i)){
|
||||
|
||||
else
|
||||
throw Error(format("path `%1%' is not a valid state/store path") % *i);
|
||||
else{
|
||||
printMsg(lvlError, format("CHECK-STATE '%1%'") % *i);
|
||||
|
||||
//TODO also get its deriver???? for if its component is gone !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
printMsg(lvlError, format("STAGE X"));
|
||||
printMsg(lvlError, format("STAGE Y"));
|
||||
|
||||
if (gcKeepOutputs) {
|
||||
/* Hmz, identical to storePathRequisites in nix-store. */
|
||||
|
|
@ -529,7 +531,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
for (DerivationOutputs::iterator j = drv.outputs.begin();
|
||||
j != drv.outputs.end(); ++j)
|
||||
if (store->isValidPath(j->second.path))
|
||||
computeFSClosure(j->second.path, livePaths, true, false, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO KEEP STATE
|
||||
computeFSClosure(j->second.path, livePaths, true, true, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WHAT ARE (STATE) OUTPUTS ????????????
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -544,7 +546,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
PathSet tempRoots;
|
||||
FDs fds;
|
||||
readTempRoots(tempRoots, fds);
|
||||
|
||||
|
||||
/* Close the temporary roots. Note that we *cannot* do this in
|
||||
readTempRoots(), because there we may not have all locks yet,
|
||||
meaning that an invalid path can become valid (and thus add to
|
||||
|
|
@ -567,24 +569,37 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
that is not currently in in `livePaths' or `tempRootsClosed'
|
||||
can be deleted. */
|
||||
|
||||
/* Read the Nix store directory to find all currently existing
|
||||
///TODO Calc get shared paths //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
//for: livePaths, tempRootsClosed
|
||||
PathSet sharedWith_livePaths_tempRootsClosed;
|
||||
|
||||
/* Read the Nix store and state directory's to find all currently existing
|
||||
paths. */
|
||||
PathSet storePathSet;
|
||||
PathSet statePathSet;
|
||||
|
||||
if (action != gcDeleteSpecific) {
|
||||
|
||||
Paths entries = readDirectory(nixStore);
|
||||
Paths stateEntries = readDirectory(nixStoreState);
|
||||
|
||||
for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
|
||||
storePathSet.insert(canonPath(nixStore + "/" + *i));
|
||||
|
||||
for (Paths::iterator i = stateEntries.begin(); i != stateEntries.end(); ++i)
|
||||
statePathSet.insert(canonPath(nixStoreState + "/" + *i));
|
||||
|
||||
} else {
|
||||
for (PathSet::iterator i = pathsToDelete.begin();
|
||||
i != pathsToDelete.end(); ++i)
|
||||
{
|
||||
assertStorePath(*i);
|
||||
assertStorePath(*i); //TODO ASSERTSTATEPATH !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
storePathSet.insert(*i);
|
||||
}
|
||||
}
|
||||
|
||||
/* Topologically sort them under the `referrers' relation. That
|
||||
is, a < b iff a is in referrers(b). This gives us the order in
|
||||
is, a < b if a is in referrers(b). This gives us the order in
|
||||
which things can be deleted safely. */
|
||||
/* !!! when we have multiple output paths per derivation, this
|
||||
will not work anymore because we get cycles. */
|
||||
|
|
@ -654,6 +669,95 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
|
|||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/* Try to delete state paths. */
|
||||
/* Topologically sort them under the `referrers' relation. That
|
||||
is, a < b if a is in referrers(b). This gives us the order in
|
||||
which things can be deleted safely. */
|
||||
/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! when we have multiple output paths per derivation, this
|
||||
will not work anymore because we get cycles. */
|
||||
Paths statePaths;
|
||||
for (PathSet::iterator i = statePathSet.begin(); i != statePathSet.end(); ++i)
|
||||
statePaths.push_back(*i);
|
||||
|
||||
|
||||
//
|
||||
|
||||
for (Paths::iterator i = statePaths.begin(); i != statePaths.end(); ++i) {
|
||||
|
||||
debug(format("considering deletion of state path `%1%'") % *i);
|
||||
|
||||
if (livePaths.find(*i) != livePaths.end()) {
|
||||
if (action == gcDeleteSpecific)
|
||||
throw Error(format("cannot delete state path `%1%' since it is still alive") % *i);
|
||||
debug(format("live state path `%1%'") % *i);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
|
||||
debug(format("temporary root `%1%'") % *i);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (sharedWith_livePaths_tempRootsClosed.find(*i) != sharedWith_livePaths_tempRootsClosed.end()) {
|
||||
debug(format("State path `%1%' is shared with another live path") % *i);
|
||||
continue;
|
||||
}
|
||||
|
||||
debug(format("dead path `%1%'") % *i);
|
||||
result.insert(*i);
|
||||
|
||||
/* If just returning the set of dead paths, we also return the
|
||||
space that would be freed if we deleted them. */
|
||||
if (action == gcReturnDead)
|
||||
bytesFreed += computePathSize(*i);
|
||||
|
||||
if (action == gcDeleteDead || action == gcDeleteSpecific) {
|
||||
|
||||
//TODO !!!!!!!!!!!!!!!!!!!!!! state locks
|
||||
/*
|
||||
#ifndef __CYGWIN__
|
||||
AutoCloseFD fdLock;
|
||||
|
||||
/* Only delete a lock file if we can acquire a write lock
|
||||
on it. That means that it's either stale, or the
|
||||
process that created it hasn't locked it yet. In the
|
||||
latter case the other process will detect that we
|
||||
deleted the lock, and retry (see pathlocks.cc). * /
|
||||
if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
|
||||
fdLock = openLockFile(*i, false);
|
||||
if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
|
||||
debug(format("skipping active lock `%1%'") % *i);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
*/
|
||||
|
||||
if (!pathExists(*i)) continue;
|
||||
|
||||
printMsg(lvlInfo, format("deleting state path `%1%'") % *i);
|
||||
|
||||
/* Okay, it's safe to delete. */
|
||||
try {
|
||||
unsigned long long freed;
|
||||
//deleteFromState(*i, freed); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUT BACK ON
|
||||
bytesFreed += freed;
|
||||
} catch (PathInUse & e) {
|
||||
printMsg(lvlError, format("warning: %1%") % e.msg());
|
||||
}
|
||||
|
||||
/*
|
||||
#ifndef __CYGWIN__
|
||||
if (fdLock != -1)
|
||||
/* Write token to stale (deleted) lock file. * /
|
||||
writeFull(fdLock, (const unsigned char *) "d", 1);
|
||||
#endif
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -288,7 +288,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter)
|
|||
}
|
||||
|
||||
|
||||
static void _canonicalisePathMetaData(const Path & path)
|
||||
static void _canonicalisePathMetaData(const Path & path, bool recurse)
|
||||
{
|
||||
checkInterrupt();
|
||||
|
||||
|
|
@ -337,17 +337,17 @@ static void _canonicalisePathMetaData(const Path & path)
|
|||
|
||||
}
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
if (recurse && S_ISDIR(st.st_mode)) {
|
||||
Strings names = readDirectory(path);
|
||||
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
|
||||
_canonicalisePathMetaData(path + "/" + *i);
|
||||
_canonicalisePathMetaData(path + "/" + *i, true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void canonicalisePathMetaData(const Path & path)
|
||||
{
|
||||
_canonicalisePathMetaData(path);
|
||||
_canonicalisePathMetaData(path, true);
|
||||
|
||||
/* On platforms that don't have lchown(), the top-level path can't
|
||||
be a symlink, since we can't change its ownership. */
|
||||
|
|
@ -1118,7 +1118,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
|
|||
consistent metadata. */
|
||||
Transaction txn(nixDB);
|
||||
addTempRoot(path);
|
||||
if (!isValidPath(path))
|
||||
if (!isValidPathTxn(txn, path))
|
||||
throw Error(format("path `%1%' is not valid") % path);
|
||||
|
||||
HashAndWriteSink hashAndWriteSink(sink);
|
||||
|
|
@ -1418,7 +1418,7 @@ void verifyStore(bool checkContents)
|
|||
}
|
||||
else {
|
||||
PathSet references;
|
||||
queryXReferencesTxn(txn, *i, references, true, -1); //TODO
|
||||
queryXReferencesTxn(txn, *i, references, true, 0); //TODO
|
||||
for (PathSet::iterator j = references.begin();
|
||||
j != references.end(); ++j)
|
||||
{
|
||||
|
|
@ -1843,6 +1843,125 @@ void LocalStore::revertToRevision(const Path & statePath, const unsigned int rev
|
|||
}
|
||||
|
||||
|
||||
typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
|
||||
|
||||
|
||||
static void makeWritable(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||
throw SysError(format("changing writability of `%1%'") % path);
|
||||
}
|
||||
|
||||
|
||||
static void hashAndLink(bool dryRun, HashToPath & hashToPath,
|
||||
OptimiseStats & stats, const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||
|
||||
/* Sometimes SNAFUs can cause files in the Nix store to be
|
||||
modified, in particular when running programs as root under
|
||||
NixOS (example: $fontconfig/var/cache being modified). Skip
|
||||
those files. */
|
||||
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
|
||||
printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
|
||||
return;
|
||||
}
|
||||
|
||||
/* We can hard link regular files and symlinks. */
|
||||
if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
|
||||
|
||||
/* Hash the file. Note that hashPath() returns the hash over
|
||||
the NAR serialisation, which includes the execute bit on
|
||||
the file. Thus, executable and non-executable files with
|
||||
the same contents *won't* be linked (which is good because
|
||||
otherwise the permissions would be screwed up).
|
||||
|
||||
Also note that if `path' is a symlink, then we're hashing
|
||||
the contents of the symlink (i.e. the result of
|
||||
readlink()), not the contents of the target (which may not
|
||||
even exist). */
|
||||
Hash hash = hashPath(htSHA256, path);
|
||||
stats.totalFiles++;
|
||||
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
|
||||
|
||||
std::pair<Path, ino_t> prevPath = hashToPath[hash];
|
||||
|
||||
if (prevPath.first == "") {
|
||||
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Yes! We've seen a file with the same contents. Replace
|
||||
the current file with a hard link to that file. */
|
||||
stats.sameContents++;
|
||||
if (prevPath.second == st.st_ino) {
|
||||
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
|
||||
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
|
||||
|
||||
Path tempLink = (format("%1%.tmp-%2%-%3%")
|
||||
% path % getpid() % rand()).str();
|
||||
|
||||
/* Make the containing directory writable, but only if
|
||||
it's not the store itself (we don't want or need to
|
||||
mess with its permissions). */
|
||||
bool mustToggle = !isStorePath(path);
|
||||
if (mustToggle) makeWritable(dirOf(path));
|
||||
|
||||
if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
|
||||
throw SysError(format("cannot link `%1%' to `%2%'")
|
||||
% tempLink % prevPath.first);
|
||||
|
||||
/* Atomically replace the old file with the new hard link. */
|
||||
if (rename(tempLink.c_str(), path.c_str()) == -1)
|
||||
throw SysError(format("cannot rename `%1%' to `%2%'")
|
||||
% tempLink % path);
|
||||
|
||||
/* Make the directory read-only again and reset its
|
||||
timestamp back to 0. */
|
||||
if (mustToggle) _canonicalisePathMetaData(dirOf(path), false);
|
||||
|
||||
} else
|
||||
printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
|
||||
|
||||
stats.filesLinked++;
|
||||
stats.bytesFreed += st.st_size;
|
||||
}
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
Strings names = readDirectory(path);
|
||||
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
|
||||
hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
|
||||
{
|
||||
HashToPath hashToPath;
|
||||
|
||||
Paths paths;
|
||||
PathSet validPaths;
|
||||
nixDB.enumTable(noTxn, dbValidPaths, paths);
|
||||
|
||||
for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
|
||||
addTempRoot(*i);
|
||||
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
|
||||
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
|
||||
hashAndLink(dryRun, hashToPath, stats, *i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
|
||||
static void upgradeStore07()
|
||||
{
|
||||
|
|
@ -1912,7 +2031,7 @@ static void upgradeStore07()
|
|||
}
|
||||
|
||||
PathSet prevReferences;
|
||||
queryXReferencesTxn(txn, path, prevReferences, true, -1);
|
||||
queryXReferencesTxn(txn, path, prevReferences, true, 0);
|
||||
if (prevReferences.size() > 0 && references != prevReferences)
|
||||
printMsg(lvlError, format("warning: conflicting references for `%1%'") % path);
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,20 @@ const int nixSchemaVersion = 4;
|
|||
extern string drvsLogDir;
|
||||
|
||||
|
||||
struct OptimiseStats
|
||||
{
|
||||
unsigned long totalFiles;
|
||||
unsigned long sameContents;
|
||||
unsigned long filesLinked;
|
||||
unsigned long long bytesFreed;
|
||||
OptimiseStats()
|
||||
{
|
||||
totalFiles = sameContents = filesLinked = 0;
|
||||
bytesFreed = 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class LocalStore : public StoreAPI
|
||||
{
|
||||
private:
|
||||
|
|
@ -125,6 +139,10 @@ public:
|
|||
void shareState(const Path & from, const Path & to, const bool snapshot);
|
||||
|
||||
void unShareState(const Path & path, const bool branch, const bool restoreOld);
|
||||
|
||||
/* Optimise the disk space usage of the Nix store by hard-linking
|
||||
files with the same contents. */
|
||||
void optimiseStore(bool dryRun, OptimiseStats & stats);
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ void computeFSClosure(const Path & path, PathSet & paths, const bool & withCompo
|
|||
computeFSClosureTxn(noTxn, path, paths, withComponents, withState, revision, flipDirection);
|
||||
}
|
||||
|
||||
/*
|
||||
* Notice that the incoming 'paths' parameter can already be filled and only needs to be expanded
|
||||
* Notice that we CANNOT change the order of the existing paths
|
||||
*/
|
||||
void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & paths, const bool & withComponents, const bool & withState, const int revision, bool flipDirection)
|
||||
{
|
||||
PathSet allPaths;
|
||||
|
|
@ -45,27 +49,23 @@ void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & p
|
|||
if(!withComponents && !withState)
|
||||
throw Error(format("Useless call to computeFSClosure, at leat withComponents or withState must be true"));
|
||||
|
||||
//TODO HOW CAN THESE PATHS ALREADY BE VALID SOMETIMES ..... ?????????????????????
|
||||
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
|
||||
if ( !isValidPathTxn(txn, *i) && !isValidStatePathTxn(txn, *i) )
|
||||
throw Error(format("Not a state or store path: ") % *i);
|
||||
|
||||
//if withState is false, we filter out all state paths
|
||||
if( withComponents && !withState ){
|
||||
//if withComponents, we add all component paths
|
||||
if( withComponents ){
|
||||
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
|
||||
if ( isValidPathTxn(txn, *i) )
|
||||
paths.insert(*i);
|
||||
}
|
||||
//if withComponents is false, we filter out all component paths
|
||||
else if( !withComponents && withState ){
|
||||
|
||||
//if withState, we add all state paths
|
||||
if( withState ){
|
||||
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
|
||||
if ( isValidStatePathTxn(txn, *i) )
|
||||
paths.insert(*i);
|
||||
}
|
||||
//all
|
||||
else{
|
||||
paths = allPaths;
|
||||
}
|
||||
}
|
||||
|
||||
void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet & paths, const int revision, const bool & flipDirection)
|
||||
|
|
@ -88,7 +88,7 @@ void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet
|
|||
queryXReferencesTxn(txn, path, references, true, revision);
|
||||
queryXReferencesTxn(txn, path, stateReferences, false, revision);
|
||||
}
|
||||
|
||||
|
||||
PathSet allReferences;
|
||||
allReferences = pathSets_union(references, stateReferences);
|
||||
|
||||
|
|
|
|||
|
|
@ -399,7 +399,7 @@ void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path
|
|||
}
|
||||
}
|
||||
|
||||
// ******************************************************************* DB FUNCTIONS
|
||||
// ******************************************************* DB FUNCTIONS ********************************************************************
|
||||
|
||||
/* State specific db functions */
|
||||
|
||||
|
|
|
|||
|
|
@ -1216,6 +1216,19 @@ string getCallingUserName()
|
|||
}
|
||||
*/
|
||||
|
||||
/* adds the second PathSet after the first, but removing doubles from the second (union)
|
||||
* (We assume the first PathSet has no duplicates)
|
||||
* UNTESTED !!!!!!!!!!!!!!
|
||||
*/
|
||||
void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths)
|
||||
{
|
||||
for (PathSet::iterator i = newPaths.begin(); i != newPaths.end(); ++i){
|
||||
if (originalPaths.find(*i) != originalPaths.end())
|
||||
continue;
|
||||
originalPaths.insert(*i);
|
||||
}
|
||||
}
|
||||
|
||||
//merges two PathSets into one, removing doubles (union)
|
||||
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
|
||||
{
|
||||
|
|
@ -1225,11 +1238,11 @@ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
|
|||
|
||||
set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); //Also available: set_symmetric_difference and set_intersection
|
||||
|
||||
PathSet diff;
|
||||
PathSet unionPaths;
|
||||
for(unsigned int i=0; i<setResult.size(); i++)
|
||||
diff.insert(setResult[i]);
|
||||
unionPaths.insert(setResult[i]);
|
||||
|
||||
return diff;
|
||||
return unionPaths;
|
||||
}
|
||||
|
||||
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths)
|
||||
|
|
|
|||
|
|
@ -314,9 +314,12 @@ unsigned int getTimeStamp();
|
|||
|
||||
//string getCallingUserName();
|
||||
|
||||
/* TODO */
|
||||
/* Merges two PathSets into one, removing doubles (union) */
|
||||
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2);
|
||||
|
||||
/* TODO UNTESTED !!!!!!!!!!!!!! */
|
||||
void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths);
|
||||
|
||||
/* TODO */
|
||||
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths);
|
||||
|
||||
|
|
|
|||
|
|
@ -437,20 +437,6 @@ void run(Strings args)
|
|||
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/store/hbxqq4d67j2y21xzp7yp01qjfkcjjbc7-hellohardcodedstateworld-1.0"));
|
||||
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"));
|
||||
|
||||
PathSet p1;
|
||||
PathSet p2;
|
||||
PathSet p3;
|
||||
PathSet p4;
|
||||
p1.insert("a");
|
||||
p1.insert("c"); //old
|
||||
p1.insert("b");
|
||||
p2.insert("b");
|
||||
p2.insert("a");
|
||||
p2.insert("cc"); //new
|
||||
p2.insert("x"); //new
|
||||
pathSets_difference(p1,p2,p3,p4);
|
||||
pathSets_union(p1,p2);
|
||||
|
||||
store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test");
|
||||
|
||||
store = openStore();
|
||||
|
|
@ -575,19 +561,33 @@ void run(Strings args)
|
|||
|
||||
copyContents("/nix/state/fwir6jlqygy90zadnx95zryfa8918qac-hellohardcodedstateworld-1.0-test/", "/home/wouterdb/tmp/aa/"); //TODO !!!!!!!!!!!!!!!!!!!
|
||||
|
||||
|
||||
|
||||
store = openStore();
|
||||
Hash hash;
|
||||
PathSet references;
|
||||
PathSet stateReferences;
|
||||
registerValidPath(noTxn, "/nix/store/sjnaisdpqyckc75c3mjz4msi9s1662cw-hellohardcodedstateworld-1.0", hash, references, stateReferences, "/nix/store/y7abzzpiknzps5kjb4qvdxvlhvxl6slj-hellohardcodedstateworld_import-1.0.drv", 0);
|
||||
|
||||
store = openStore();
|
||||
store->isStateComponent("/nix/state/h72n75fx3c40aslbbp41vmj9gxl2skp1-hellohardcodedstateworld-1.0-test");
|
||||
|
||||
PathSet p1;
|
||||
PathSet p2;
|
||||
PathSet p3;
|
||||
PathSet p4;
|
||||
p1.insert("a");
|
||||
p1.insert("c"); //old
|
||||
p1.insert("b");
|
||||
p2.insert("b");
|
||||
p2.insert("a");
|
||||
p2.insert("cc"); //new
|
||||
p2.insert("x"); //new
|
||||
//pathSets_difference(p1,p2,p3,p4);
|
||||
pathSets_union_ordered(p1,p2);
|
||||
|
||||
throw Error(format("aaa"));
|
||||
return;
|
||||
|
||||
//*/
|
||||
|
||||
*/
|
||||
|
||||
|
||||
/* test */
|
||||
|
||||
if(args.size() == 1 && ( *(args.begin()) == "--help" || *(args.begin()) == "--statehelp")){
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ Operations:
|
|||
|
||||
--init: initialise the Nix database
|
||||
--verify: verify Nix structures
|
||||
--optimise: optimise the Nix store by hard-linking identical files
|
||||
|
||||
--version: output version information
|
||||
--help: display help
|
||||
|
|
|
|||
|
|
@ -470,6 +470,13 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
|
|||
}
|
||||
|
||||
|
||||
static string showBytes(unsigned long long bytes)
|
||||
{
|
||||
return (format("%d bytes (%.2f MiB)")
|
||||
% bytes % (bytes / (1024.0 * 1024.0))).str();
|
||||
}
|
||||
|
||||
|
||||
struct PrintFreed
|
||||
{
|
||||
bool show, dryRun;
|
||||
|
|
@ -481,9 +488,9 @@ struct PrintFreed
|
|||
if (show)
|
||||
cout << format(
|
||||
(dryRun
|
||||
? "%d bytes would be freed (%.2f MiB)\n"
|
||||
: "%d bytes freed (%.2f MiB)\n"))
|
||||
% bytesFreed % (bytesFreed / (1024.0 * 1024.0));
|
||||
? "%1% would be freed\n"
|
||||
: "%1% freed\n"))
|
||||
% showBytes(bytesFreed);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -618,6 +625,46 @@ static void opVerify(Strings opFlags, Strings opArgs)
|
|||
}
|
||||
|
||||
|
||||
|
||||
static void showOptimiseStats(OptimiseStats & stats)
|
||||
{
|
||||
printMsg(lvlError,
|
||||
format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
|
||||
% showBytes(stats.bytesFreed)
|
||||
% stats.filesLinked
|
||||
% stats.sameContents
|
||||
% stats.totalFiles);
|
||||
}
|
||||
|
||||
|
||||
/* Optimise the disk space usage of the Nix store by hard-linking
|
||||
files with the same contents. */
|
||||
static void opOptimise(Strings opFlags, Strings opArgs)
|
||||
{
|
||||
if (!opArgs.empty())
|
||||
throw UsageError("no arguments expected");
|
||||
|
||||
bool dryRun = false;
|
||||
|
||||
for (Strings::iterator i = opFlags.begin();
|
||||
i != opFlags.end(); ++i)
|
||||
if (*i == "--dry-run") dryRun = true;
|
||||
else throw UsageError(format("unknown flag `%1%'") % *i);
|
||||
|
||||
LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
|
||||
if (!store2) throw Error("you don't have sufficient rights to use --optimise");
|
||||
|
||||
OptimiseStats stats;
|
||||
try {
|
||||
store2->optimiseStore(dryRun, stats);
|
||||
} catch (...) {
|
||||
showOptimiseStats(stats);
|
||||
throw;
|
||||
}
|
||||
showOptimiseStats(stats);
|
||||
}
|
||||
|
||||
|
||||
/* Scan the arguments; find the operation, set global flags, put all
|
||||
other flags in a list, and put all other arguments in another
|
||||
list. */
|
||||
|
|
@ -663,6 +710,8 @@ void run(Strings args)
|
|||
op = opInit;
|
||||
else if (arg == "--verify")
|
||||
op = opVerify;
|
||||
else if (arg == "--optimise")
|
||||
op = opOptimise;
|
||||
else if (arg == "--add-root") {
|
||||
if (i == args.end())
|
||||
throw UsageError("`--add-root requires an argument");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue