nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-26 20:20:58 +01:00
Code Activity

In the middle of adding state references to derivations and the db...

Browse source
This commit is contained in:
Wouter den Breejen 2007-06-18 19:54:31 +00:00
parent 5e59387d40
commit bdecf3bdbc
16 changed files with 190 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

30
src/libstore/build.cc
View file

@ -1621,6 +1621,7 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
void DerivationGoal::computeClosure()
{
map<Path, PathSet> allReferences;
map<Path, PathSet> allStateReferences;
map<Path, Hash> contentHashes;
//TODO MOVE THIS TO A PLACE THAT ALSO GETS CALLED WHEN WE DONT NEED TO BUILD ANYTHING
@ -1646,8 +1647,7 @@ void DerivationGoal::computeClosure()
if (lstat(path.c_str(), &st) == -1)
throw SysError(format("getting attributes of path `%1%'") % path);
startNest(nest, lvlTalkative,
format("scanning for references inside `%1%'") % path);
startNest(nest, lvlTalkative, format("scanning for component and state references inside `%1%'") % path);
/* Check that fixed-output derivations produced the right
outputs (i.e., the content hash should match the specified
@ -1689,10 +1689,20 @@ void DerivationGoal::computeClosure()
/* For this output path, find the references to other paths contained in it. */
PathSet references = scanForReferences(path, allPaths);
/* For this state-output path, find the references to other paths contained in it. */
Path statePath = drv.stateOutputs.find("state")->second.statepath;
PathSet state_references = scanForReferences(statePath, allPaths);
references = mergePathSets(references, state_references);
/* For this state-output path, find the references to other paths contained in it.
* Get the state paths (instead of out paths) from all components, and then call
* scanForStateReferences().
*/
PathSet allStatePaths;
for (PathSet::const_iterator i = allPaths.begin(); i != allPaths.end(); i++){
Path componentPath = *i;
if(store->isStateComponent(componentPath)){
PathSet stateRefs = queryDerivers(noTxn, componentPath ,"*",getCallingUserName());
stateRefs = mergePathSets(stateRefs, allStatePaths);
}
}
PathSet stateReferences = scanForStateReferences(path, allStatePaths);
/* For debugging, print out the referenced and unreferenced
paths. */
@ -1707,6 +1717,8 @@ void DerivationGoal::computeClosure()
}
allReferences[path] = references;
allStateReferences[path] = stateReferences;
/* If the derivation specifies an `allowedReferences'
attribute (containing a list of paths that the output may
@ -1745,6 +1757,7 @@ void DerivationGoal::computeClosure()
registerValidPath(txn, i->second.path,
contentHashes[i->second.path],
allReferences[i->second.path],
allStateReferences[i->second.path],
drvPath);
}
txn.commit();
@ -1873,6 +1886,9 @@ private:
/* Outgoing references for this path. */
PathSet references;
/* Outgoing state references for this path. */ //TODO CHECK THIS ENTIRE FILE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
PathSet stateReferences;
/* Pipe for the substitute's standard output/error. */
Pipe logPipe;
@ -2145,7 +2161,7 @@ void SubstitutionGoal::finished()
Transaction txn;
createStoreTransaction(txn);
registerValidPath(txn, storePath, contentHash,
references, sub.deriver);
references, stateReferences, sub.deriver);
txn.commit();
outputLock->setDeletion(true);

8
src/libstore/derivations.cc
View file

@ -27,11 +27,15 @@ Path writeDerivation(const Derivation & drv, const string & name)
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
//TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
PathSet stateReferences;
string suffix = name + drvExtension;
string contents = atPrint(unparseDerivation(drv));
return readOnlyMode
? computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references);
? computeStorePathForText(suffix, contents, references, stateReferences)
: store->addTextToStore(suffix, contents, references, stateReferences);
}

222
src/libstore/local-store.cc
View file

@ -44,6 +44,15 @@ static TableId dbValidPaths = 0;
paths. */
static TableId dbReferences = 0;
/* dbReferences :: Path -> [Path]
This table lists the outgoing file system state references for each
output path that has been built by a Nix derivation. These are
found by scanning the path for the hash components of input
paths. */
static TableId dbStateReferences = 0;
/* dbReferrers :: Path -> Path
This table is just the reverse mapping of dbReferences. This table
@ -83,6 +92,13 @@ static TableId dbDerivers = 0;
static TableId dbStateCounters = 0;
/* dbStateCounters :: Path -> String
This table lists the all the state managed components, TODO we could store the entire DRV in here in the future
*/
static TableId dbStateInfo = 0;
bool Substitute::operator == (const Substitute & sub) const
{
return program == sub.program
@ -143,10 +159,12 @@ LocalStore::LocalStore(bool reserveSpace)
}
dbValidPaths = nixDB.openTable("validpaths");
dbReferences = nixDB.openTable("references");
dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */
dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ //TODO ADD STATE REFERERS?
dbSubstitutes = nixDB.openTable("substitutes");
dbDerivers = nixDB.openTable("derivers");
dbStateInfo = nixDB.openTable("stateinfo");
dbStateCounters = nixDB.openTable("statecounters");
dbStateReferences = nixDB.openTable("references_state");
int curSchema = 0;
Path schemaFN = nixDBPath + "/schema";
@ -365,7 +383,7 @@ void queryReferences(const Transaction & txn,
{
Paths references2;
//WOUTER EDIT
//WOUTER EDIT TODO
if (!isRealisablePath(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
@ -406,18 +424,18 @@ void setDeriver(const Transaction & txn, const Path & storePath, const Path & de
if (!isRealisablePath(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
Derivation drv = derivationFromPath(deriver); //Redirect if its a state component
if (drv.outputs.size() != 0)
addStateDeriver(txn, storePath, deriver);
else
Derivation drv = derivationFromPath(deriver);
if (drv.outputs.size() != 0){ //Redirect if its a state component
addStateDeriver(txn, storePath, deriver);
}
else{
nixDB.setString(txn, dbDerivers, storePath, deriver);
}
}
void addStateDeriver(const Transaction & txn, const Path & storePath, const Path & deriver)
{
printMsg(lvlError, format("Adding State Derivers into DB %1%") % deriver);
assertStorePath(storePath);
if (deriver == "") return;
assertStorePath(deriver);
@ -432,22 +450,44 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path
PathSet updatedDerivers = mergeNewDerivationIntoList(storePath, deriver, currentDerivers, true);
Strings data;
for (PathSet::iterator i = updatedDerivers.begin(); i != updatedDerivers.end(); ++i){
string deriver = *i;
//TODO Remove obsolete check
assertStorePath(deriver);
Derivation drv = derivationFromPath(deriver);
if (drv.outputs.size() == 0) //Error if its not a state component
throw Error(format("path `%1%' is a NOT state Path at addStateDeriver") % storePath);
//TODO Remove obsolete check
data.push_back(deriver);
}
nixDB.setStrings(txn, dbDerivers, storePath, data); //update the db.
for (PathSet::iterator i = updatedDerivers.begin(); i != updatedDerivers.end(); ++i) //Convert Paths to Strings
data.push_back(*i);
nixDB.setStrings(txn, dbDerivers, storePath, data); //update the derivers db.
nixDB.setString(txn, dbStateInfo, storePath, ""); //update the dbinfo db. (maybe TODO)
}
//TODO Add and ..
bool isStateComponent(const Path & storePath)
{
store->isValidPath(storePath);
string deriver;
string data;
return nixDB.queryString(noTxn, dbStateInfo, storePath, data); //update the dbinfo db. (maybe TODO)
}
bool LocalStore::isStateComponent(const Path & storePath)
{
return nix::isStateComponent(storePath);
}
//TODO Add and ..
bool isStateDrv(const Path & drvPath)
{
Derivation drv = derivationFromPath(drvPath); //maybe redirect the out path to isStateComponent?
if (drv.outputs.size() != 0)
return true;
else
return false;
}
bool LocalStore::isStateDrv(const Path & isStateDrv)
{
return nix::isStateDrv(isStateDrv);
}
Path queryDeriver(const Transaction & txn, const Path & storePath)
{
@ -455,10 +495,9 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
throw Error(format("path `%1%' is not valid") % storePath);
Path deriver;
bool b = nixDB.queryString(txn, dbDerivers, storePath, deriver);
Derivation drv = derivationFromPath(deriver); //Redirect if its a state component
Derivation drv = derivationFromPath(deriver);
if (drv.outputs.size() != 0)
throw Error(format("This deriver `%1%' is a state deriver, u should use queryDerivers instead of queryDeriver") % deriver);
@ -491,12 +530,12 @@ PathSet queryDerivers(const Transaction & txn, const Path & storePath, const str
string getIdentifier = drv.stateOutputs.find("state")->second.stateIdentifier;
string getUser = drv.stateOutputs.find("state")->second.username;
//printMsg(lvlError, format("queryDerivers '%1%' '%2%' '%3%' '%4%' '%5%'") % derivationpath % getIdentifier % identifier % getUser % user);
if( (getIdentifier == identifier || identifier == "*") && (getUser == user || user == "*") )
filtereddata.insert(derivationpath);
}
return filtereddata;
}
@ -650,7 +689,8 @@ Hash LocalStore::queryPathHash(const Path & path)
void registerValidPath(const Transaction & txn,
const Path & path, const Hash & hash, const PathSet & references,
const Path & path, const Hash & hash,
const PathSet & references, const PathSet & stateReferences,
const Path & deriver)
{
ValidPathInfo info;
@ -748,7 +788,7 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
canonicalisePathMetaData(dstPath);
Transaction txn(nixDB);
registerValidPath(txn, dstPath, h, PathSet(), "");
registerValidPath(txn, dstPath, h, PathSet(), PathSet(), "");
txn.commit();
}
@ -760,9 +800,9 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
Path LocalStore::addTextToStore(const string & suffix, const string & s,
const PathSet & references)
const PathSet & references, const PathSet & stateReferences)
{
Path dstPath = computeStorePathForText(suffix, s, references);
Path dstPath = computeStorePathForText(suffix, s, references, stateReferences);
addTempRoot(dstPath);
@ -779,8 +819,7 @@ Path LocalStore::addTextToStore(const string & suffix, const string & s,
canonicalisePathMetaData(dstPath);
Transaction txn(nixDB);
registerValidPath(txn, dstPath,
hashPath(htSHA256, dstPath), references, "");
registerValidPath(txn, dstPath, hashPath(htSHA256, dstPath), references, stateReferences, "");
txn.commit();
}
@ -919,6 +958,9 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
Path dstPath = readStorePath(hashAndReadSource);
PathSet references = readStorePaths(hashAndReadSource);
//TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
PathSet stateReferences;
Path deriver = readString(hashAndReadSource);
if (deriver != "") assertStorePath(deriver);
@ -983,7 +1025,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
here. */
if (!isValidPath(deriver)) deriver = "";
registerValidPath(txn, dstPath,
hashPath(htSHA256, dstPath), references, deriver);
hashPath(htSHA256, dstPath), references, stateReferences, deriver);
txn.commit();
}
@ -1315,125 +1357,21 @@ PathSet mergeNewDerivationIntoList(const Path & storepath, const Path & newdrv,
string getIdentifier = getdrv.stateOutputs.find("state")->second.stateIdentifier;
string getUser = getdrv.stateOutputs.find("state")->second.username;
if( !(identifier == getIdentifier && getUser == user) ) //only insert if it doenst already exist
newdrvs.insert(drv);
else{
if(identifier == getIdentifier && getUser == user) //only insert if it doenst already exist
{
if(deleteDrvs){
printMsg(lvlError, format("Deleting decrepated state derivation: %1% with identifier %2% and user %3%") % drv % identifier % user);
deletePath(drv); //Deletes the DRV from DISK!
}
}
}
else
newdrvs.insert(drv);
}
newdrvs.insert(newdrv);
return newdrvs;
}
//TODO INCLUDE USERNAME + IDENTIFIER, the can be multiple derivations for the same component
//TODO add mergeNewDerivationIntoList
void updateAllStateDerivations()
{
//call AddStateDerivation !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/*
Transaction txn(nixDB);
// Filter out all decrepated derivations, e.g. with a decrepated timestamp /
Strings unique_paths;
Strings keys;
nixDB.enumTable(txn, dbUpdatedDerivations, keys);
for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
{
string drv_key = *i; //the key is the derivation
Strings data;
nixDB.queryStrings(txn, dbUpdatedDerivations, drv_key, data);
string path = data.front();
bool exists = false;
for (Strings::iterator j = unique_paths.begin(); j != unique_paths.end(); ++j)
{
string unique_path = *j;
if(path == unique_path)
exists = true;
}
if(!exists)
unique_paths.push_back(path);
}
// Now we merge the two sets of: storepath -> [derivations] together into the db /
for (Strings::iterator i = unique_paths.begin(); i != unique_paths.end(); ++i)
{
string storepath = *i;
printMsg(lvlError, format("Unique: %1%") % storepath);
Path drvPath = queryDeriver(txn, storepath);
Path originalDerivation = drvPath;
Path newDerivation = drvPath; //the new drv path first equals the old one until a new one is found
int timestamp = 0;
//Get the (multiple) derivations of references
Strings keys;
nixDB.enumTable(txn, dbUpdatedDerivations, keys);
for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
{
string drv_key = *i; //the key is the derivation
Strings data;
nixDB.queryStrings(txn, dbUpdatedDerivations, drv_key, data);
string getstorepath = data.front();
data.pop_front();
string gettimestamp = data.front();
if(storepath == getstorepath){
int gettimestamp_i;
string2Int(gettimestamp, gettimestamp_i);
if(gettimestamp_i == timestamp)
throw Error(format("Error! Multiple changes at store path %4% at the same time: derivations: `%1%' and `%2%' with timestamp `%3%'") % newDerivation % drv_key % timestamp % storepath);
if(timestamp == 0 || gettimestamp_i > timestamp){ //we choose the new derivation as the latest submitted derivation
//printMsg(lvlError, format("Replacing at store path %4% the old drv (%1%) with new drv (%2%) with timestamp: %3%") % newDerivation % drv_key % gettimestamp_i % storepath);
newDerivation = drv_key;
timestamp = gettimestamp_i;
}
//Always Remove the old updatelink in the dbUpdatedDerivations
nixDB.delPair(txn, dbUpdatedDerivations, drv_key);
}
}
if(originalDerivation != newDerivation) //only update if neccecary
{
//Replace the old deriver link in the derivers database (TODO, maybe delete old deriver path????????)
setDeriver(txn, storepath, newDerivation);
//Call the stateUpdate function for the new derivation? Yes since this function is called at build time
printMsg(lvlError, format("Calling new state drv %1% for storepath %2%") % newDerivation % storepath);
//TODO check wheter we update before or after ??
//TODO
}
}
txn.commit();
*/
}
void LocalStore::updateAllStateDerivations()
{
nix::updateAllStateDerivations();
}
/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */

11
src/libstore/local-store.hh
View file

@ -54,7 +54,7 @@ public:
PathFilter & filter = defaultPathFilter);
Path addTextToStore(const string & suffix, const string & s,
const PathSet & references);
const PathSet & references, const PathSet & stateReferences);
void exportPath(const Path & path, bool sign,
Sink & sink);
@ -82,9 +82,9 @@ public:
PathSet getStateReferencesClosure(const Path & path);
void addUpdatedStateDerivation(const Path & newdrv, const Path & storepath);
void updateAllStateDerivations();
bool isStateComponent(const Path & path);
bool isStateDrv(const Path & drvpath);
};
@ -109,7 +109,8 @@ void clearSubstitutes();
of the file system contents of the path. The hash must be a
SHA-256 hash. */
void registerValidPath(const Transaction & txn,
const Path & path, const Hash & hash, const PathSet & references,
const Path & path, const Hash & hash,
const PathSet & references, const PathSet & stateReferences,
const Path & deriver);
struct ValidPathInfo

12
src/libstore/references.cc
View file

@ -152,5 +152,17 @@ PathSet scanForReferences(const string & path, const PathSet & paths)
return found;
}
/* A wrapper for now, but we may extend */
PathSet scanForStateReferences(const string & path, const PathSet & statePaths)
{
return scanForReferences(path, statePaths);
}
PathSet scanForALLReferences(const string & path)
{
PathSet p;
throw Error("TODO");
return p;
}
}

4
src/libstore/references.hh
View file

@ -6,6 +6,10 @@
namespace nix {
PathSet scanForReferences(const Path & path, const PathSet & refs);
PathSet scanForStateReferences(const string & path, const PathSet & statePaths);
PathSet scanForALLReferences(const string & path);
}

13
src/libstore/remote-store.cc
View file

@ -230,12 +230,13 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
Path RemoteStore::addTextToStore(const string & suffix, const string & s,
const PathSet & references)
const PathSet & references, const PathSet & stateReferences)
{
writeInt(wopAddTextToStore, to);
writeString(suffix, to);
writeString(s, to);
writeStringSet(references, to);
writeStringSet(stateReferences, to);
processStderr();
Path path = readStorePath(from);
@ -395,10 +396,14 @@ PathSet RemoteStore::getStateReferencesClosure(const Path & path)
return empty;
}
//TODO
void RemoteStore::updateAllStateDerivations()
bool RemoteStore::isStateComponent(const Path & path)
{
return false;
}
bool RemoteStore::isStateDrv(const Path & drvpath)
{
return false;
}

8
src/libstore/remote-store.hh
View file

@ -42,7 +42,7 @@ public:
PathFilter & filter = defaultPathFilter);
Path addTextToStore(const string & suffix, const string & s,
const PathSet & references);
const PathSet & references, const PathSet & stateReferences);
void exportPath(const Path & path, bool sign,
Sink & sink);
@ -69,9 +69,11 @@ public:
vector<int> getStatePathsInterval(const PathSet & statePaths);
PathSet getStateReferencesClosure(const Path & path);
void updateAllStateDerivations();
bool isStateComponent(const Path & path);
bool isStateDrv(const Path & drvpath);
private:
AutoCloseFD fdSocket;

7
src/libstore/store-api.cc
View file

@ -179,7 +179,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
Path computeStorePathForText(const string & suffix, const string & s,
const PathSet & references)
const PathSet & references, const PathSet & stateReferences)
{
Hash hash = hashString(htSHA256, s);
/* Stuff the references (if any) into the type. This is a bit
@ -190,6 +190,11 @@ Path computeStorePathForText(const string & suffix, const string & s,
type += ":";
type += *i;
}
for (PathSet::const_iterator i = stateReferences.begin(); i != stateReferences.end(); ++i) {
type += ":";
type += *i;
}
return makeStorePath(type, hash, suffix);
}

10
src/libstore/store-api.hh
View file

@ -92,7 +92,7 @@ public:
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual Path addTextToStore(const string & suffix, const string & s,
const PathSet & references) = 0;
const PathSet & references, const PathSet & stateReferences) = 0;
/* Export a store path, that is, create a NAR dump of the store
path and append its references and its deriver. Optionally, a
@ -192,8 +192,12 @@ public:
virtual PathSet getStateReferencesClosure(const Path & path) = 0;
/* TODO */
virtual void updateAllStateDerivations() = 0;
virtual bool isStateComponent(const Path & path) = 0;
/* TODO */
virtual bool isStateDrv(const Path & drvpath) = 0;
};
@ -252,7 +256,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
affected), but it has some backwards compatibility issues (the
hashing scheme changes), so I'm not doing that for now. */
Path computeStorePathForText(const string & suffix, const string & s,
const PathSet & references);
const PathSet & references, const PathSet & stateReferences);
/* Remove the temporary roots file for this process. Any temporary

8
src/libstore/store-state.cc
View file

@ -21,7 +21,7 @@ void updatedStateDerivation(Path storePath)
//We dont remove the old .svn folders
//New repostorys are created by createStateDirs
printMsg(lvlError, format("Resetting state drv settings like repositorys"));
printMsg(lvlTalkative, format("Resetting state drv settings like repositorys"));
//Create a repository for this state location
@ -60,7 +60,7 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De
executeAndPrintShellCommand("mkdir -p " + repos, "mkdir");
if(IsDirectory(repos))
printMsg(lvlError, format("Repos %1% already exists, so we use that repository") % repos);
printMsg(lvlTalkative, format("Repos %1% already exists, so we use that repository") % repos);
else
executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit than ??
@ -68,14 +68,14 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De
intervalPaths.insert(statePath);
}
printMsg(lvlError, format("Adding state subdir: %1% to %2% from repository %3%") % thisdir % fullstatedir % repos);
printMsg(lvlTalkative, format("Adding state subdir: %1% to %2% from repository %3%") % thisdir % fullstatedir % repos);
if(IsDirectory(fullstatedir + "/.svn/")){
string checkoutcommand = svnbin + " checkout file://" + repos + " " + fullstatedir;
executeAndPrintShellCommand(checkoutcommand, "svn"); //TODO checkout as user
}
else
printMsg(lvlError, format("Statedir %1% already exists, so dont check out its repository again") % fullstatedir);
printMsg(lvlTalkative, format("Statedir %1% already exists, so dont check out its repository again") % fullstatedir);
}
//Initialize the counters for the statePaths that have an interval to 0