diff --git a/install_full.sh b/install_full.sh index f27f9d677..f180111a8 100755 --- a/install_full.sh +++ b/install_full.sh @@ -1,5 +1,7 @@ #! /bin/sh -e +make clean + export nixstatepath=/nixstate2/nix export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal diff --git a/scripts/optimise-store.pl b/scripts/optimise-store.pl deleted file mode 100755 index 41557e6d1..000000000 --- a/scripts/optimise-store.pl +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/perl -w - -use strict; -use File::Basename; - - -my @paths = ("/nix/store"); - - -print "hashing...\n"; - -my $hashList = "/tmp/nix-optimise-hash-list"; - -system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0 - or die "cannot hash store files"; - - -print "sorting by hash...\n"; - -system("sort $hashList > $hashList.sorted") == 0 - or die "cannot sort list"; - - -sub atomicLink { - my $target = shift; - my $new = shift; - my $tmpNew = "${new}_optimise.$$"; - - # Make the directory writable temporarily. - my $dir = dirname $new; - my @st = stat $dir or die; - - chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!"; - - link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!"; - - rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!"; - - chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!"; - utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!"; -} - - -print "hard-linking...\n"; - -open LIST, "<$hashList.sorted" or die; - -my $prevFile; -my $prevHash; -my $prevInode; -my $prevExec; - -my $totalSpace = 0; -my $savedSpace = 0; - -while () { - /^([0-9a-f]*)\s+(.*)$/ or die; - my $curFile = $2; - my $curHash = $1; - - my @st = stat $curFile or die; - next if ($st[2] & 0222) != 0; # skip writable files - - my $fileSize = $st[7]; - $totalSpace += $fileSize; - my $isExec = ($st[2] & 0111) == 0111; - - if (defined $prevHash && $curHash eq $prevHash - && $prevExec == $isExec) - { - - if ($st[1] != $prevInode) { - print "$curFile = $prevFile\n"; - atomicLink $prevFile, $curFile; - $savedSpace += $fileSize; - } - - } else { - $prevFile = $curFile; - $prevHash = $curHash; - $prevInode = $st[1]; - $prevExec = ($st[2] & 0111) == 0111; - } -} - -print "total space = $totalSpace\n"; -print "saved space = $savedSpace\n"; -my $savings = ($savedSpace / $totalSpace) * 100.0; -print "savings = $savings %\n"; - -close LIST; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8288a7e14..5d9dfdc00 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -452,11 +452,11 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, /* Acquire the global GC root. This prevents a) New roots from being added. b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + AutoCloseFD fdGCLock = openGCLock(ltWrite); //TODO !!!!!!!!!!!!!!!!!!! GET STATE LOCKS !!!!!!!!!!!!!!!!!!!!!!!!!!!!! /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true); //TODO Also find state roots --> nah ? TODO include sharing of state. + Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true); PathSet roots; for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i) @@ -480,9 +480,10 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, PathSet livePaths; for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i){ printMsg(lvlError, format("CHECK '%1%'") % *i); - computeFSClosure(canonPath(*i), livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? + computeFSClosure(canonPath(*i), livePaths, true, true, 0); } - + + printMsg(lvlError, format("STAGE X")); if (gcKeepDerivations) { for (PathSet::iterator i = livePaths.begin(); @@ -496,28 +497,29 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j) // We send each drv to computeFSClosure if (*j != "" && store->isValidPath(*j)) - computeFSClosure(*j, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? + computeFSClosure(*j, livePaths, true, true, 0); } else if (store->isValidPath(*i)){ printMsg(lvlError, format("CHECK-STORE '%1%'") % *i); Path deriver = store->queryDeriver(*i); - printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver); + //printMsg(lvlError, format("CHECK-STORE DRV '%1%'") % deriver); /* Note that the deriver need not be valid (e.g., if we previously ran the collector with `gcKeepDerivations' turned off). */ if (deriver != "" && store->isValidPath(deriver)) - computeFSClosure(deriver, livePaths, true, false, 0); //TODO !!!!!!!!!! ALSO STATEPATHS TRUE??? + computeFSClosure(deriver, livePaths, true, true, 0); } - - //else if (store->isValidStatePath(*i)){ - - else - throw Error(format("path `%1%' is not a valid state/store path") % *i); + else{ + printMsg(lvlError, format("CHECK-STATE '%1%'") % *i); + + //TODO also get its deriver???? for if its component is gone !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + } } } - printMsg(lvlError, format("STAGE X")); + printMsg(lvlError, format("STAGE Y")); if (gcKeepOutputs) { /* Hmz, identical to storePathRequisites in nix-store. */ @@ -529,7 +531,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, for (DerivationOutputs::iterator j = drv.outputs.begin(); j != drv.outputs.end(); ++j) if (store->isValidPath(j->second.path)) - computeFSClosure(j->second.path, livePaths, true, false, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO KEEP STATE + computeFSClosure(j->second.path, livePaths, true, true, 0); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WHAT ARE (STATE) OUTPUTS ???????????? } } @@ -544,7 +546,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, PathSet tempRoots; FDs fds; readTempRoots(tempRoots, fds); - + /* Close the temporary roots. Note that we *cannot* do this in readTempRoots(), because there we may not have all locks yet, meaning that an invalid path can become valid (and thus add to @@ -567,24 +569,37 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, that is not currently in in `livePaths' or `tempRootsClosed' can be deleted. */ - /* Read the Nix store directory to find all currently existing + ///TODO Calc get shared paths //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //for: livePaths, tempRootsClosed + PathSet sharedWith_livePaths_tempRootsClosed; + + /* Read the Nix store and state directory's to find all currently existing paths. */ PathSet storePathSet; + PathSet statePathSet; + if (action != gcDeleteSpecific) { + Paths entries = readDirectory(nixStore); + Paths stateEntries = readDirectory(nixStoreState); + for (Paths::iterator i = entries.begin(); i != entries.end(); ++i) storePathSet.insert(canonPath(nixStore + "/" + *i)); + + for (Paths::iterator i = stateEntries.begin(); i != stateEntries.end(); ++i) + statePathSet.insert(canonPath(nixStoreState + "/" + *i)); + } else { for (PathSet::iterator i = pathsToDelete.begin(); i != pathsToDelete.end(); ++i) { - assertStorePath(*i); + assertStorePath(*i); //TODO ASSERTSTATEPATH !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! storePathSet.insert(*i); } } /* Topologically sort them under the `referrers' relation. That - is, a < b iff a is in referrers(b). This gives us the order in + is, a < b if a is in referrers(b). This gives us the order in which things can be deleted safely. */ /* !!! when we have multiple output paths per derivation, this will not work anymore because we get cycles. */ @@ -654,6 +669,95 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, #endif } } + + /* Try to delete state paths. */ + /* Topologically sort them under the `referrers' relation. That + is, a < b if a is in referrers(b). This gives us the order in + which things can be deleted safely. */ + /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! when we have multiple output paths per derivation, this + will not work anymore because we get cycles. */ + Paths statePaths; + for (PathSet::iterator i = statePathSet.begin(); i != statePathSet.end(); ++i) + statePaths.push_back(*i); + + + // + + for (Paths::iterator i = statePaths.begin(); i != statePaths.end(); ++i) { + + debug(format("considering deletion of state path `%1%'") % *i); + + if (livePaths.find(*i) != livePaths.end()) { + if (action == gcDeleteSpecific) + throw Error(format("cannot delete state path `%1%' since it is still alive") % *i); + debug(format("live state path `%1%'") % *i); + continue; + } + + if (tempRootsClosed.find(*i) != tempRootsClosed.end()) { + debug(format("temporary root `%1%'") % *i); + continue; + } + + if (sharedWith_livePaths_tempRootsClosed.find(*i) != sharedWith_livePaths_tempRootsClosed.end()) { + debug(format("State path `%1%' is shared with another live path") % *i); + continue; + } + + debug(format("dead path `%1%'") % *i); + result.insert(*i); + + /* If just returning the set of dead paths, we also return the + space that would be freed if we deleted them. */ + if (action == gcReturnDead) + bytesFreed += computePathSize(*i); + + if (action == gcDeleteDead || action == gcDeleteSpecific) { + +//TODO !!!!!!!!!!!!!!!!!!!!!! state locks +/* +#ifndef __CYGWIN__ + AutoCloseFD fdLock; + + /* Only delete a lock file if we can acquire a write lock + on it. That means that it's either stale, or the + process that created it hasn't locked it yet. In the + latter case the other process will detect that we + deleted the lock, and retry (see pathlocks.cc). * / + if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") { + fdLock = openLockFile(*i, false); + if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) { + debug(format("skipping active lock `%1%'") % *i); + continue; + } + } +#endif +*/ + + if (!pathExists(*i)) continue; + + printMsg(lvlInfo, format("deleting state path `%1%'") % *i); + + /* Okay, it's safe to delete. */ + try { + unsigned long long freed; + //deleteFromState(*i, freed); //TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! PUT BACK ON + bytesFreed += freed; + } catch (PathInUse & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + } + +/* +#ifndef __CYGWIN__ + if (fdLock != -1) + /* Write token to stale (deleted) lock file. * / + writeFull(fdLock, (const unsigned char *) "d", 1); +#endif +*/ + + } + + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 34eed5275..ee2732910 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -288,7 +288,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter) } -static void _canonicalisePathMetaData(const Path & path) +static void _canonicalisePathMetaData(const Path & path, bool recurse) { checkInterrupt(); @@ -337,17 +337,17 @@ static void _canonicalisePathMetaData(const Path & path) } - if (S_ISDIR(st.st_mode)) { + if (recurse && S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); for (Strings::iterator i = names.begin(); i != names.end(); ++i) - _canonicalisePathMetaData(path + "/" + *i); + _canonicalisePathMetaData(path + "/" + *i, true); } } void canonicalisePathMetaData(const Path & path) { - _canonicalisePathMetaData(path); + _canonicalisePathMetaData(path, true); /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ @@ -1118,7 +1118,7 @@ void LocalStore::exportPath(const Path & path, bool sign, consistent metadata. */ Transaction txn(nixDB); addTempRoot(path); - if (!isValidPath(path)) + if (!isValidPathTxn(txn, path)) throw Error(format("path `%1%' is not valid") % path); HashAndWriteSink hashAndWriteSink(sink); @@ -1418,7 +1418,7 @@ void verifyStore(bool checkContents) } else { PathSet references; - queryXReferencesTxn(txn, *i, references, true, -1); //TODO + queryXReferencesTxn(txn, *i, references, true, 0); //TODO for (PathSet::iterator j = references.begin(); j != references.end(); ++j) { @@ -1843,6 +1843,125 @@ void LocalStore::revertToRevision(const Path & statePath, const unsigned int rev } +typedef std::map > HashToPath; + + +static void makeWritable(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of `%1%'") % path); +} + + +static void hashAndLink(bool dryRun, HashToPath & hashToPath, + OptimiseStats & stats, const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); + return; + } + + /* We can hard link regular files and symlinks. */ + if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + + /* Hash the file. Note that hashPath() returns the hash over + the NAR serialisation, which includes the execute bit on + the file. Thus, executable and non-executable files with + the same contents *won't* be linked (which is good because + otherwise the permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing + the contents of the symlink (i.e. the result of + readlink()), not the contents of the target (which may not + even exist). */ + Hash hash = hashPath(htSHA256, path); + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + + std::pair prevPath = hashToPath[hash]; + + if (prevPath.first == "") { + hashToPath[hash] = std::pair(path, st.st_ino); + return; + } + + /* Yes! We've seen a file with the same contents. Replace + the current file with a hard link to that file. */ + stats.sameContents++; + if (prevPath.second == st.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); + return; + } + + if (!dryRun) { + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); + + Path tempLink = (format("%1%.tmp-%2%-%3%") + % path % getpid() % rand()).str(); + + /* Make the containing directory writable, but only if + it's not the store itself (we don't want or need to + mess with its permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) + throw SysError(format("cannot link `%1%' to `%2%'") + % tempLink % prevPath.first); + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) + throw SysError(format("cannot rename `%1%' to `%2%'") + % tempLink % path); + + /* Make the directory read-only again and reset its + timestamp back to 0. */ + if (mustToggle) _canonicalisePathMetaData(dirOf(path), false); + + } else + printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); + } +} + + +void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) +{ + HashToPath hashToPath; + + Paths paths; + PathSet validPaths; + nixDB.enumTable(noTxn, dbValidPaths, paths); + + for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) { + addTempRoot(*i); + if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ + startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); + hashAndLink(dryRun, hashToPath, stats, *i); + } +} + + /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */ static void upgradeStore07() { @@ -1912,7 +2031,7 @@ static void upgradeStore07() } PathSet prevReferences; - queryXReferencesTxn(txn, path, prevReferences, true, -1); + queryXReferencesTxn(txn, path, prevReferences, true, 0); if (prevReferences.size() > 0 && references != prevReferences) printMsg(lvlError, format("warning: conflicting references for `%1%'") % path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1a702fa3..13f8c644d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -21,6 +21,20 @@ const int nixSchemaVersion = 4; extern string drvsLogDir; +struct OptimiseStats +{ + unsigned long totalFiles; + unsigned long sameContents; + unsigned long filesLinked; + unsigned long long bytesFreed; + OptimiseStats() + { + totalFiles = sameContents = filesLinked = 0; + bytesFreed = 0; + } +}; + + class LocalStore : public StoreAPI { private: @@ -125,6 +139,10 @@ public: void shareState(const Path & from, const Path & to, const bool snapshot); void unShareState(const Path & path, const bool branch, const bool restoreOld); + + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(bool dryRun, OptimiseStats & stats); }; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c81a31487..72535880f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -37,6 +37,10 @@ void computeFSClosure(const Path & path, PathSet & paths, const bool & withCompo computeFSClosureTxn(noTxn, path, paths, withComponents, withState, revision, flipDirection); } +/* + * Notice that the incoming 'paths' parameter can already be filled and only needs to be expanded + * Notice that we CANNOT change the order of the existing paths + */ void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & paths, const bool & withComponents, const bool & withState, const int revision, bool flipDirection) { PathSet allPaths; @@ -45,27 +49,23 @@ void computeFSClosureTxn(const Transaction & txn, const Path & path, PathSet & p if(!withComponents && !withState) throw Error(format("Useless call to computeFSClosure, at leat withComponents or withState must be true")); - //TODO HOW CAN THESE PATHS ALREADY BE VALID SOMETIMES ..... ????????????????????? for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) if ( !isValidPathTxn(txn, *i) && !isValidStatePathTxn(txn, *i) ) throw Error(format("Not a state or store path: ") % *i); - //if withState is false, we filter out all state paths - if( withComponents && !withState ){ + //if withComponents, we add all component paths + if( withComponents ){ for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) if ( isValidPathTxn(txn, *i) ) paths.insert(*i); } - //if withComponents is false, we filter out all component paths - else if( !withComponents && withState ){ + + //if withState, we add all state paths + if( withState ){ for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i) if ( isValidStatePathTxn(txn, *i) ) paths.insert(*i); } - //all - else{ - paths = allPaths; - } } void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet & paths, const int revision, const bool & flipDirection) @@ -88,7 +88,7 @@ void computeFSClosureRecTxn(const Transaction & txn, const Path & path, PathSet queryXReferencesTxn(txn, path, references, true, revision); queryXReferencesTxn(txn, path, stateReferences, false, revision); } - + PathSet allReferences; allReferences = pathSets_union(references, stateReferences); diff --git a/src/libstore/store-state.cc b/src/libstore/store-state.cc index dd9faa9a8..27c5dbba4 100644 --- a/src/libstore/store-state.cc +++ b/src/libstore/store-state.cc @@ -399,7 +399,7 @@ void scanAndUpdateAllReferencesRecusivelyTxn(const Transaction & txn, const Path } } -// ******************************************************************* DB FUNCTIONS +// ******************************************************* DB FUNCTIONS ******************************************************************** /* State specific db functions */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f44f2ed8..65627ec71 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1216,6 +1216,19 @@ string getCallingUserName() } */ +/* adds the second PathSet after the first, but removing doubles from the second (union) + * (We assume the first PathSet has no duplicates) + * UNTESTED !!!!!!!!!!!!!! + */ +void pathSets_union_ordered(PathSet & originalPaths, const PathSet & newPaths) +{ + for (PathSet::iterator i = newPaths.begin(); i != newPaths.end(); ++i){ + if (originalPaths.find(*i) != originalPaths.end()) + continue; + originalPaths.insert(*i); + } +} + //merges two PathSets into one, removing doubles (union) PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2) { @@ -1225,11 +1238,11 @@ PathSet pathSets_union(const PathSet & paths1, const PathSet & paths2) set_union(vector1.begin(), vector1.end(),vector2.begin(), vector2.end(), back_inserter(setResult)); //Also available: set_symmetric_difference and set_intersection - PathSet diff; + PathSet unionPaths; for(unsigned int i=0; iscanForAllReferences("/nix/state/i06flm2ahq5s0x3633z30dnav9f1wkb5-hellohardcodedstateworld-dep1-1.0-test"); store = openStore(); @@ -575,19 +561,33 @@ void run(Strings args) copyContents("/nix/state/fwir6jlqygy90zadnx95zryfa8918qac-hellohardcodedstateworld-1.0-test/", "/home/wouterdb/tmp/aa/"); //TODO !!!!!!!!!!!!!!!!!!! - - - store = openStore(); Hash hash; PathSet references; PathSet stateReferences; registerValidPath(noTxn, "/nix/store/sjnaisdpqyckc75c3mjz4msi9s1662cw-hellohardcodedstateworld-1.0", hash, references, stateReferences, "/nix/store/y7abzzpiknzps5kjb4qvdxvlhvxl6slj-hellohardcodedstateworld_import-1.0.drv", 0); + + store = openStore(); + store->isStateComponent("/nix/state/h72n75fx3c40aslbbp41vmj9gxl2skp1-hellohardcodedstateworld-1.0-test"); + + PathSet p1; + PathSet p2; + PathSet p3; + PathSet p4; + p1.insert("a"); + p1.insert("c"); //old + p1.insert("b"); + p2.insert("b"); + p2.insert("a"); + p2.insert("cc"); //new + p2.insert("x"); //new + //pathSets_difference(p1,p2,p3,p4); + pathSets_union_ordered(p1,p2); + + throw Error(format("aaa")); return; - - //*/ + */ - /* test */ if(args.size() == 1 && ( *(args.begin()) == "--help" || *(args.begin()) == "--statehelp")){ diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index c43a9fcd3..a338fe7d1 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -21,6 +21,7 @@ Operations: --init: initialise the Nix database --verify: verify Nix structures + --optimise: optimise the Nix store by hard-linking identical files --version: output version information --help: display help diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f74a030ae..41fa00a2d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -470,6 +470,13 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) } +static string showBytes(unsigned long long bytes) +{ + return (format("%d bytes (%.2f MiB)") + % bytes % (bytes / (1024.0 * 1024.0))).str(); +} + + struct PrintFreed { bool show, dryRun; @@ -481,9 +488,9 @@ struct PrintFreed if (show) cout << format( (dryRun - ? "%d bytes would be freed (%.2f MiB)\n" - : "%d bytes freed (%.2f MiB)\n")) - % bytesFreed % (bytesFreed / (1024.0 * 1024.0)); + ? "%1% would be freed\n" + : "%1% freed\n")) + % showBytes(bytesFreed); } }; @@ -618,6 +625,46 @@ static void opVerify(Strings opFlags, Strings opArgs) } + +static void showOptimiseStats(OptimiseStats & stats) +{ + printMsg(lvlError, + format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total") + % showBytes(stats.bytesFreed) + % stats.filesLinked + % stats.sameContents + % stats.totalFiles); +} + + +/* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ +static void opOptimise(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + + bool dryRun = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--dry-run") dryRun = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + LocalStore * store2(dynamic_cast(store.get())); + if (!store2) throw Error("you don't have sufficient rights to use --optimise"); + + OptimiseStats stats; + try { + store2->optimiseStore(dryRun, stats); + } catch (...) { + showOptimiseStats(stats); + throw; + } + showOptimiseStats(stats); +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -663,6 +710,8 @@ void run(Strings args) op = opInit; else if (arg == "--verify") op = opVerify; + else if (arg == "--optimise") + op = opOptimise; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root requires an argument");