nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-26 04:00:59 +01:00
Code Activity

Merged to R9439. Fixed a computeFSClosure bug. The state garbage colletor basically works, Missing items: State locks, shared state and Topological sort

Browse source
This commit is contained in:
Wouter den Breejen 2007-10-10 15:55:00 +00:00
parent 7d82fd16e9
commit 16410fc714
12 changed files with 372 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

2
install_full.sh
View file

@ -1,5 +1,7 @@
#! /bin/sh -e #! /bin/sh -e
make clean
export nixstatepath=/nixstate2/nix export nixstatepath=/nixstate2/nix
export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal

91
scripts/optimise-store.pl
View file

@ -1,91 +0,0 @@
#! /usr/bin/perl -w
use strict;
use File::Basename;
my @paths = ("/nix/store");
print "hashing...\n";
my $hashList = "/tmp/nix-optimise-hash-list";
system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0
or die "cannot hash store files";
print "sorting by hash...\n";
system("sort $hashList > $hashList.sorted") == 0
or die "cannot sort list";
sub atomicLink {
my $target = shift;
my $new = shift;
my $tmpNew = "${new}_optimise.$$";
# Make the directory writable temporarily.
my $dir = dirname $new;
my @st = stat $dir or die;
chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!";
link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!";
rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!";
chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!";
utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!";
}
print "hard-linking...\n";
open LIST, "<$hashList.sorted" or die;
my $prevFile;
my $prevHash;
my $prevInode;
my $prevExec;
my $totalSpace = 0;
my $savedSpace = 0;
while (<LIST>) {
/^([0-9a-f]*)\s+(.*)$/ or die;
my $curFile = $2;
my $curHash = $1;
my @st = stat $curFile or die;
next if ($st[2] & 0222) != 0; # skip writable files
my $fileSize = $st[7];
$totalSpace += $fileSize;
my $isExec = ($st[2] & 0111) == 0111;
if (defined $prevHash && $curHash eq $prevHash
&& $prevExec == $isExec)
{
if ($st[1] != $prevInode) {
print "$curFile = $prevFile\n";
atomicLink $prevFile, $curFile;
$savedSpace += $fileSize;
}
} else {
$prevFile = $curFile;
$prevHash = $curHash;
$prevInode = $st[1];
$prevExec = ($st[2] & 0111) == 0111;
}
}
print "total space = $totalSpace\n";
print "saved space = $savedSpace\n";
my $savings = ($savedSpace / $totalSpace) * 100.0;
print "savings = $savings %\n";
close LIST;

140
src/libstore/gc.cc
View file

@ -452,11 +452,11 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
/* Acquire the global GC root. This prevents /* Acquire the global GC root. This prevents
a) New roots from being added. a) New roots from being added.
b) Processes from creating new temporary root files. */ b) Processes from creating new temporary root files. */
AutoCloseFD fdGCLock = openGCLock(ltWrite); AutoCloseFD fdGCLock = openGCLock(ltWrite); //TODO !!!!!!!!!!!!!!!!!!! GET STATE LOCKS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/* Find the roots. Since we've grabbed the GC lock, the set of /* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */ permanent roots cannot increase now. */
Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true); //TODO Also find state roots --> nah ? TODO include sharing of state. Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
PathSet roots; PathSet roots;
for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i) for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
@ -480,9 +480,10 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
PathSet livePaths; PathSet livePaths;
for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){ for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){
printMsg(lvlError, format("CHECK '%1%'") % *i); printMsg(lvlError, format("CHECK '%1%'") % *i);
computeFSClosure(canonPath(*i), livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? computeFSClosure(canonPath(*i), livePaths, true, true, 0);
} }
printMsg(lvlError, format("STAGE X"));
if (gcKeepDerivations) { if (gcKeepDerivations) {
for (PathSet::iterator i = livePaths.begin(); for (PathSet::iterator i = livePaths.begin();
@ -496,28 +497,29 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j) for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j)
// We send each drv to computeFSClosure // We send each drv to computeFSClosure
if (*j != "" && store->isValidPath(*j)) if (*j != "" && store->isValidPath(*j))
computeFSClosure(*j, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? computeFSClosure(*j, livePaths, true, true, 0);
} }
else if (store->isValidPath(*i)){ else if (store->isValidPath(*i)){
printMsg(lvlError, format("CHECK-STORE '%1%'") % *i); printMsg(lvlError, format("CHECK-STORE '%1%'") % *i);
Path deriver = store->queryDeriver(*i); Path deriver = store->queryDeriver(*i);
printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver); //printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver);
/* Note that the deriver need not be valid (e.g., if we /* Note that the deriver need not be valid (e.g., if we
previously ran the collector with `gcKeepDerivations' previously ran the collector with `gcKeepDerivations'
turned off). */ turned off). */
if (deriver != "" && store->isValidPath(deriver)) if (deriver != "" && store->isValidPath(deriver))
computeFSClosure(deriver, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? computeFSClosure(deriver, livePaths, true, true, 0);
} }
else{
//else if (store->isValidStatePath(*i)){ printMsg(lvlError, format("CHECK-STATE '%1%'") % *i);
else //TODO also get its deriver???? for if its component is gone !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
throw Error(format("path `%1%' is not a valid state/store path") % *i);
}
} }
} }
printMsg(lvlError, format("STAGE X")); printMsg(lvlError, format("STAGE Y"));
if (gcKeepOutputs) { if (gcKeepOutputs) {
/* Hmz, identical to storePathRequisites in nix-store. */ /* Hmz, identical to storePathRequisites in nix-store. */
@ -529,7 +531,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
for (DerivationOutputs::iterator j = drv.outputs.begin(); for (DerivationOutputs::iterator j = drv.outputs.begin();
j != drv.outputs.end(); ++j) j != drv.outputs.end(); ++j)
if (store->isValidPath(j->second.path)) if (store->isValidPath(j->second.path))
computeFSClosure(j->second.path, livePaths, true, false, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO KEEP STATE computeFSClosure(j->second.path, livePaths, true, true, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WHAT ARE (STATE) OUTPUTS ????????????
} }
} }
@ -544,7 +546,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
PathSet tempRoots; PathSet tempRoots;
FDs fds; FDs fds;
readTempRoots(tempRoots, fds); readTempRoots(tempRoots, fds);
/* Close the temporary roots. Note that we *cannot* do this in /* Close the temporary roots. Note that we *cannot* do this in
readTempRoots(), because there we may not have all locks yet, readTempRoots(), because there we may not have all locks yet,
meaning that an invalid path can become valid (and thus add to meaning that an invalid path can become valid (and thus add to
@ -567,24 +569,37 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
that is not currently in in `livePaths' or `tempRootsClosed' that is not currently in in `livePaths' or `tempRootsClosed'
can be deleted. */ can be deleted. */
/* Read the Nix store directory to find all currently existing ///TODO Calc get shared paths //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//for: livePaths, tempRootsClosed
PathSet sharedWith_livePaths_tempRootsClosed;
/* Read the Nix store and state directory's to find all currently existing
paths. */ paths. */
PathSet storePathSet; PathSet storePathSet;
PathSet statePathSet;
if (action != gcDeleteSpecific) { if (action != gcDeleteSpecific) {
Paths entries = readDirectory(nixStore); Paths entries = readDirectory(nixStore);
Paths stateEntries = readDirectory(nixStoreState);
for (Paths::iterator i = entries.begin(); i != entries.end(); ++i) for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
storePathSet.insert(canonPath(nixStore + "/" + *i)); storePathSet.insert(canonPath(nixStore + "/" + *i));
for (Paths::iterator i = stateEntries.begin(); i != stateEntries.end(); ++i)
statePathSet.insert(canonPath(nixStoreState + "/" + *i));
} else { } else {
for (PathSet::iterator i = pathsToDelete.begin(); for (PathSet::iterator i = pathsToDelete.begin();
i != pathsToDelete.end(); ++i) i != pathsToDelete.end(); ++i)
{ {
assertStorePath(*i); assertStorePath(*i); //TODO ASSERTSTATEPATH !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
storePathSet.insert(*i); storePathSet.insert(*i);
} }
} }
/* Topologically sort them under the `referrers' relation. That /* Topologically sort them under the `referrers' relation. That
is, a < b iff a is in referrers(b). This gives us the order in is, a < b if a is in referrers(b). This gives us the order in
which things can be deleted safely. */ which things can be deleted safely. */
/* !!! when we have multiple output paths per derivation, this /* !!! when we have multiple output paths per derivation, this
will not work anymore because we get cycles. */ will not work anymore because we get cycles. */
@ -654,6 +669,95 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
#endif #endif
} }
} }
/* Try to delete state paths. */
/* Topologically sort them under the `referrers' relation. That
is, a < b if a is in referrers(b). This gives us the order in
which things can be deleted safely. */
/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! when we have multiple output paths per derivation, this
will not work anymore because we get cycles. */
Paths statePaths;
for (PathSet::iterator i = statePathSet.begin(); i != statePathSet.end(); ++i)
statePaths.push_back(*i);
//
for (Paths::iterator i = statePaths.begin(); i != statePaths.end(); ++i) {
debug(format("considering deletion of state path `%1%'") % *i);
if (livePaths.find(*i) != livePaths.end()) {
if (action == gcDeleteSpecific)
throw Error(format("cannot delete state path `%1%' since it is still alive") % *i);
debug(format("live state path `%1%'") % *i);
continue;
}
if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
debug(format("temporary root `%1%'") % *i);
continue;
}
if (sharedWith_livePaths_tempRootsClosed.find(*i) != sharedWith_livePaths_tempRootsClosed.end()) {
debug(format("State path `%1%' is shared with another live path") % *i);
continue;
}
debug(format("dead path `%1%'") % *i);
result.insert(*i);
/* If just returning the set of dead paths, we also return the
space that would be freed if we deleted them. */
if (action == gcReturnDead)
bytesFreed += computePathSize(*i);
if (action == gcDeleteDead || action == gcDeleteSpecific) {
//TODO !!!!!!!!!!!!!!!!!!!!!! state locks
/*
#ifndef __CYGWIN__
AutoCloseFD fdLock;
/* Only delete a lock file if we can acquire a write lock
on it. That means that it's either stale, or the
process that created it hasn't locked it yet. In the
latter case the other process will detect that we
deleted the lock, and retry (see pathlocks.cc). * /
if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
fdLock = openLockFile(*i, false);
if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
debug(format("skipping active lock `%1%'") % *i);
continue;
}
}
#endif
*/
if (!pathExists(*i)) continue;
printMsg(lvlInfo, format("deleting state path `%1%'") % *i);
/* Okay, it's safe to delete. */
try {
unsigned long long freed;
//deleteFromState(*i, freed); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUT BACK ON
bytesFreed += freed;
} catch (PathInUse & e) {
printMsg(lvlError, format("warning: %1%") % e.msg());
}
/*
#ifndef __CYGWIN__
if (fdLock != -1)
/* Write token to stale (deleted) lock file. * /
writeFull(fdLock, (const unsigned char *) "d", 1);
#endif
*/
}
}
} }

133
src/libstore/local-store.cc
View file

@ -288,7 +288,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter)
} }
static void _canonicalisePathMetaData(const Path & path) static void _canonicalisePathMetaData(const Path & path, bool recurse)
{ {
checkInterrupt(); checkInterrupt();
@ -337,17 +337,17 @@ static void _canonicalisePathMetaData(const Path & path)
} }
if (S_ISDIR(st.st_mode)) { if (recurse && S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path); Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i) for (Strings::iterator i = names.begin(); i != names.end(); ++i)
_canonicalisePathMetaData(path + "/" + *i); _canonicalisePathMetaData(path + "/" + *i, true);
} }
} }
void canonicalisePathMetaData(const Path & path) void canonicalisePathMetaData(const Path & path)
{ {
_canonicalisePathMetaData(path); _canonicalisePathMetaData(path, true);
/* On platforms that don't have lchown(), the top-level path can't /* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */ be a symlink, since we can't change its ownership. */
@ -1118,7 +1118,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
consistent metadata. */ consistent metadata. */
Transaction txn(nixDB); Transaction txn(nixDB);
addTempRoot(path); addTempRoot(path);
if (!isValidPath(path)) if (!isValidPathTxn(txn, path))
throw Error(format("path `%1%' is not valid") % path); throw Error(format("path `%1%' is not valid") % path);
HashAndWriteSink hashAndWriteSink(sink); HashAndWriteSink hashAndWriteSink(sink);
@ -1418,7 +1418,7 @@ void verifyStore(bool checkContents)
} }
else { else {
PathSet references; PathSet references;
queryXReferencesTxn(txn, *i, references, true, -1); //TODO queryXReferencesTxn(txn, *i, references, true, 0); //TODO
for (PathSet::iterator j = references.begin(); for (PathSet::iterator j = references.begin();
j != references.end(); ++j) j != references.end(); ++j)
{ {
@ -1843,6 +1843,125 @@ void LocalStore::revertToRevision(const Path & statePath, const unsigned int rev
} }
typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
static void makeWritable(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("changing writability of `%1%'") % path);
}
static void hashAndLink(bool dryRun, HashToPath & hashToPath,
OptimiseStats & stats, const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
/* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under
NixOS (example: $fontconfig/var/cache being modified). Skip
those files. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
return;
}
/* We can hard link regular files and symlinks. */
if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
/* Hash the file. Note that hashPath() returns the hash over
the NAR serialisation, which includes the execute bit on
the file. Thus, executable and non-executable files with
the same contents *won't* be linked (which is good because
otherwise the permissions would be screwed up).
Also note that if `path' is a symlink, then we're hashing
the contents of the symlink (i.e. the result of
readlink()), not the contents of the target (which may not
even exist). */
Hash hash = hashPath(htSHA256, path);
stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
std::pair<Path, ino_t> prevPath = hashToPath[hash];
if (prevPath.first == "") {
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
return;
}
/* Yes! We've seen a file with the same contents. Replace
the current file with a hard link to that file. */
stats.sameContents++;
if (prevPath.second == st.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
return;
}
if (!dryRun) {
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
Path tempLink = (format("%1%.tmp-%2%-%3%")
% path % getpid() % rand()).str();
/* Make the containing directory writable, but only if
it's not the store itself (we don't want or need to
mess with its permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
throw SysError(format("cannot link `%1%' to `%2%'")
% tempLink % prevPath.first);
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1)
throw SysError(format("cannot rename `%1%' to `%2%'")
% tempLink % path);
/* Make the directory read-only again and reset its
timestamp back to 0. */
if (mustToggle) _canonicalisePathMetaData(dirOf(path), false);
} else
printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
stats.filesLinked++;
stats.bytesFreed += st.st_size;
}
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
}
}
void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
{
HashToPath hashToPath;
Paths paths;
PathSet validPaths;
nixDB.enumTable(noTxn, dbValidPaths, paths);
for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
addTempRoot(*i);
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
hashAndLink(dryRun, hashToPath, stats, *i);
}
}
/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
static void upgradeStore07() static void upgradeStore07()
{ {
@ -1912,7 +2031,7 @@ static void upgradeStore07()
} }
PathSet prevReferences; PathSet prevReferences;
queryXReferencesTxn(txn, path, prevReferences, true, -1); queryXReferencesTxn(txn, path, prevReferences, true, 0);
if (prevReferences.size() > 0 && references != prevReferences) if (prevReferences.size() > 0 && references != prevReferences)
printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); printMsg(lvlError, format("warning: conflicting references for `%1%'") % path);

18
src/libstore/local-store.hh
View file

@ -21,6 +21,20 @@ const int nixSchemaVersion = 4;
extern string drvsLogDir; extern string drvsLogDir;
struct OptimiseStats
{
unsigned long totalFiles;
unsigned long sameContents;
unsigned long filesLinked;
unsigned long long bytesFreed;
OptimiseStats()
{
totalFiles = sameContents = filesLinked = 0;
bytesFreed = 0;
}
};
class LocalStore : public StoreAPI class LocalStore : public StoreAPI
{ {
private: private:
@ -125,6 +139,10 @@ public:
void shareState(const Path & from, const Path & to, const bool snapshot); void shareState(const Path & from, const Path & to, const bool snapshot);
void unShareState(const Path & path, const bool branch, const bool restoreOld); void unShareState(const Path & path, const bool branch, const bool restoreOld);
/* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */
void optimiseStore(bool dryRun, OptimiseStats & stats);
}; };

20
src/libstore/misc.cc
View file

@ -37,6 +37,10 @@ void computeFSClosure(const Path & path, PathSet & paths, const bool & withCompo
computeFSClosureTxn(noTxn, path, paths, withComponents, withState, revision, flipDirection); computeFSClosureTxn(noTxn, path, paths, withComponents, withState, revision, flipDirection);
} }
/*
* Notice that the incoming 'paths' parameter can already be filled and only needs to be expanded
* Notice that we CANNOT change the order of the existing paths
*/
void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & paths, const bool & withComponents, const bool & withState, const int revision, bool flipDirection) void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & paths, const bool & withComponents, const bool & withState, const int revision, bool flipDirection)
{ {
PathSet allPaths; PathSet allPaths;
@ -45,27 +49,23 @@ void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & p
if(!withComponents && !withState) if(!withComponents && !withState)
throw Error(format("Useless call to computeFSClosure, at leat withComponents or withState must be true")); throw Error(format("Useless call to computeFSClosure, at leat withComponents or withState must be true"));
//TODO HOW CAN THESE PATHS ALREADY BE VALID SOMETIMES ..... ?????????????????????
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
if ( !isValidPathTxn(txn, *i) && !isValidStatePathTxn(txn, *i) ) if ( !isValidPathTxn(txn, *i) && !isValidStatePathTxn(txn, *i) )
throw Error(format("Not a state or store path: ") % *i); throw Error(format("Not a state or store path: ") % *i);
//if withState is false, we filter out all state paths //if withComponents, we add all component paths
if( withComponents && !withState ){ if( withComponents ){
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
if ( isValidPathTxn(txn, *i) ) if ( isValidPathTxn(txn, *i) )
paths.insert(*i); paths.insert(*i);
} }
//if withComponents is false, we filter out all component paths
else if( !withComponents && withState ){ //if withState, we add all state paths
if( withState ){
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
if ( isValidStatePathTxn(txn, *i) ) if ( isValidStatePathTxn(txn, *i) )
paths.insert(*i); paths.insert(*i);
} }
//all
else{
paths = allPaths;
}
} }
void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet & paths, const int revision, const bool & flipDirection) void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet & paths, const int revision, const bool & flipDirection)
@ -88,7 +88,7 @@ void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet
queryXReferencesTxn(txn, path, references, true, revision); queryXReferencesTxn(txn, path, references, true, revision);
queryXReferencesTxn(txn, path, stateReferences, false, revision); queryXReferencesTxn(txn, path, stateReferences, false, revision);
} }
PathSet allReferences; PathSet allReferences;
allReferences = pathSets_union(references, stateReferences); allReferences = pathSets_union(references, stateReferences);

2
src/libstore/store-state.cc
View file

@ -399,7 +399,7 @@ void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path
} }
} }
// ******************************************************************* DB FUNCTIONS // ******************************************************* DB FUNCTIONS ********************************************************************
/* State specific db functions */ /* State specific db functions */

19
src/libutil/util.cc
View file

@ -1216,6 +1216,19 @@ string getCallingUserName()
} }
*/ */
/* adds the second PathSet after the first, but removing doubles from the second (union)
* (We assume the first PathSet has no duplicates)
* UNTESTED !!!!!!!!!!!!!!
*/
void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths)
{
for (PathSet::iterator i = newPaths.begin(); i != newPaths.end(); ++i){
if (originalPaths.find(*i) != originalPaths.end())
continue;
originalPaths.insert(*i);
}
}
//merges two PathSets into one, removing doubles (union) //merges two PathSets into one, removing doubles (union)
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2) PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
{ {
@ -1225,11 +1238,11 @@ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2)
set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); //Also available: set_symmetric_difference and set_intersection set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); //Also available: set_symmetric_difference and set_intersection
PathSet diff; PathSet unionPaths;
for(unsigned int i=0; i<setResult.size(); i++) for(unsigned int i=0; i<setResult.size(); i++)
diff.insert(setResult[i]); unionPaths.insert(setResult[i]);
return diff; return unionPaths;
} }
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths) void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths)

5
src/libutil/util.hh
View file

@ -314,9 +314,12 @@ unsigned int getTimeStamp();
//string getCallingUserName(); //string getCallingUserName();
/* TODO */ /* Merges two PathSets into one, removing doubles (union) */
PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2); PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2);
/* TODO UNTESTED !!!!!!!!!!!!!! */
void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths);
/* TODO */ /* TODO */
void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths); void pathSets_difference(const PathSet & oldpaths, const PathSet & newpaths, PathSet & addedpaths, PathSet & removedpaths);

40
src/nix-state/nix-state.cc
View file

@ -437,20 +437,6 @@ void run(Strings args)
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/store/hbxqq4d67j2y21xzp7yp01qjfkcjjbc7-hellohardcodedstateworld-1.0")); printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/store/hbxqq4d67j2y21xzp7yp01qjfkcjjbc7-hellohardcodedstateworld-1.0"));
printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2")); printMsg(lvlError, format("AA: %1%") % isStatePath("/nix/state/0qhlpz1ji4gvg3j6nk5vkcddmi3m5x1r-hellohardcodedstateworld-1.0-test2"));
PathSet p1;
PathSet p2;
PathSet p3;
PathSet p4;
p1.insert("a");
p1.insert("c"); //old
p1.insert("b");
p2.insert("b");
p2.insert("a");
p2.insert("cc"); //new
p2.insert("x"); //new
pathSets_difference(p1,p2,p3,p4);
pathSets_union(p1,p2);
store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test"); store->scanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test");
store = openStore(); store = openStore();
@ -575,19 +561,33 @@ void run(Strings args)
copyContents("/nix/state/fwir6jlqygy90zadnx95zryfa8918qac-hellohardcodedstateworld-1.0-test/", "/home/wouterdb/tmp/aa/"); //TODO !!!!!!!!!!!!!!!!!!! copyContents("/nix/state/fwir6jlqygy90zadnx95zryfa8918qac-hellohardcodedstateworld-1.0-test/", "/home/wouterdb/tmp/aa/"); //TODO !!!!!!!!!!!!!!!!!!!
store = openStore();
Hash hash; Hash hash;
PathSet references; PathSet references;
PathSet stateReferences; PathSet stateReferences;
registerValidPath(noTxn, "/nix/store/sjnaisdpqyckc75c3mjz4msi9s1662cw-hellohardcodedstateworld-1.0", hash, references, stateReferences, "/nix/store/y7abzzpiknzps5kjb4qvdxvlhvxl6slj-hellohardcodedstateworld_import-1.0.drv", 0); registerValidPath(noTxn, "/nix/store/sjnaisdpqyckc75c3mjz4msi9s1662cw-hellohardcodedstateworld-1.0", hash, references, stateReferences, "/nix/store/y7abzzpiknzps5kjb4qvdxvlhvxl6slj-hellohardcodedstateworld_import-1.0.drv", 0);
store = openStore();
store->isStateComponent("/nix/state/h72n75fx3c40aslbbp41vmj9gxl2skp1-hellohardcodedstateworld-1.0-test");
PathSet p1;
PathSet p2;
PathSet p3;
PathSet p4;
p1.insert("a");
p1.insert("c"); //old
p1.insert("b");
p2.insert("b");
p2.insert("a");
p2.insert("cc"); //new
p2.insert("x"); //new
//pathSets_difference(p1,p2,p3,p4);
pathSets_union_ordered(p1,p2);
throw Error(format("aaa"));
return; return;
//*/
*/
/* test */ /* test */
if(args.size() == 1 && ( *(args.begin()) == "--help" || *(args.begin()) == "--statehelp")){ if(args.size() == 1 && ( *(args.begin()) == "--help" || *(args.begin()) == "--statehelp")){

1
src/nix-store/help.txt
View file

@ -21,6 +21,7 @@ Operations:
--init: initialise the Nix database --init: initialise the Nix database
--verify: verify Nix structures --verify: verify Nix structures
--optimise: optimise the Nix store by hard-linking identical files
--version: output version information --version: output version information
--help: display help --help: display help

55
src/nix-store/nix-store.cc
View file

@ -470,6 +470,13 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
} }
static string showBytes(unsigned long long bytes)
{
return (format("%d bytes (%.2f MiB)")
% bytes % (bytes / (1024.0 * 1024.0))).str();
}
struct PrintFreed struct PrintFreed
{ {
bool show, dryRun; bool show, dryRun;
@ -481,9 +488,9 @@ struct PrintFreed
if (show) if (show)
cout << format( cout << format(
(dryRun (dryRun
? "%d bytes would be freed (%.2f MiB)\n" ? "%1% would be freed\n"
: "%d bytes freed (%.2f MiB)\n")) : "%1% freed\n"))
% bytesFreed % (bytesFreed / (1024.0 * 1024.0)); % showBytes(bytesFreed);
} }
}; };
@ -618,6 +625,46 @@ static void opVerify(Strings opFlags, Strings opArgs)
} }
static void showOptimiseStats(OptimiseStats & stats)
{
printMsg(lvlError,
format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
% showBytes(stats.bytesFreed)
% stats.filesLinked
% stats.sameContents
% stats.totalFiles);
}
/* Optimise the disk space usage of the Nix store by hard-linking
files with the same contents. */
static void opOptimise(Strings opFlags, Strings opArgs)
{
if (!opArgs.empty())
throw UsageError("no arguments expected");
bool dryRun = false;
for (Strings::iterator i = opFlags.begin();
i != opFlags.end(); ++i)
if (*i == "--dry-run") dryRun = true;
else throw UsageError(format("unknown flag `%1%'") % *i);
LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
if (!store2) throw Error("you don't have sufficient rights to use --optimise");
OptimiseStats stats;
try {
store2->optimiseStore(dryRun, stats);
} catch (...) {
showOptimiseStats(stats);
throw;
}
showOptimiseStats(stats);
}
/* Scan the arguments; find the operation, set global flags, put all /* Scan the arguments; find the operation, set global flags, put all
other flags in a list, and put all other arguments in another other flags in a list, and put all other arguments in another
list. */ list. */
@ -663,6 +710,8 @@ void run(Strings args)
op = opInit; op = opInit;
else if (arg == "--verify") else if (arg == "--verify")
op = opVerify; op = opVerify;
else if (arg == "--optimise")
op = opOptimise;
else if (arg == "--add-root") { else if (arg == "--add-root") {
if (i == args.end()) if (i == args.end())
throw UsageError("`--add-root requires an argument"); throw UsageError("`--add-root requires an argument");