1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-26 20:20:58 +01:00

nix-state now works, state is recursively commited (when necessary)

This commit is contained in:
Wouter den Breejen 2007-06-27 15:43:16 +00:00
parent c0dcfed3c3
commit 3d22bd50b3
17 changed files with 457 additions and 277 deletions

View file

@ -1629,6 +1629,7 @@ void DerivationGoal::computeClosure()
if(!drv.stateOutputs.find("state")->second.getCreateDirsBeforeInstall())
createStateDirs(drv.stateOutputDirs, drv.stateOutputs, drv.env);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
@ -1681,7 +1682,7 @@ void DerivationGoal::computeClosure()
format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
% path % algo % printHash(h) % printHash(h2));
}
/* Get rid of all weird permissions. */
canonicalisePathMetaData(path);
@ -1719,7 +1720,7 @@ void DerivationGoal::computeClosure()
if we could combine this with filterReferences(). */
contentHashes[path] = hashPath(htSHA256, path);
}
/* Register each output path as valid, and register the sets of
paths referenced by each of them. This is wrapped in one
database transaction to ensure that if we crash, either
@ -1737,6 +1738,8 @@ void DerivationGoal::computeClosure()
for (DerivationOutputs::iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
{
//printMsg(lvlError, format("SetValidPath: %1%") % i->second.path);
registerValidPath(txn, i->second.path,
"", //dummy statePath
contentHashes[i->second.path],
@ -1744,19 +1747,22 @@ void DerivationGoal::computeClosure()
PathSet(), //dummy stateReferences
drvPath);
}
/*
* We first register alls paths as valid, and only scan for component references.
* Now that those paths are registered as valid, we're able to call queryDeriversStatePath
*
* We already scanned for Component references in Component paths
* Now we scan in Component paths for state references
* We already scanned for [Component references in Component paths] //1
* Now we scan in [Component paths for state references] //2
*
* If state is enabled for the path we:
* scan for and state references and component references in the state path
* [scan for and state references and component references in the state path] //3,4
*/
//TODO we scan for each output, be then we do multiple scans inside for the state path .....
//TODO !!!!!!!!! we don not ONLY need to scan all outputs, but also (recursively) the component references in the state folders
for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
{
Path path = i->second.path;
@ -1770,7 +1776,7 @@ void DerivationGoal::computeClosure()
Path componentPath = *i;
if(isStateComponentTxn(txn, componentPath)){
//printMsg(lvlError, format("Scanning for state path: %1%") % (*i));
//printMsg(lvlError, format("Scanning for state path: %1%") % componentPath);
PathSet stateRefs = queryDeriversStatePath(txn, componentPath ,"*",getCallingUserName());
allStatePaths = mergePathSets(stateRefs, allStatePaths);
}
@ -1783,7 +1789,7 @@ void DerivationGoal::computeClosure()
if(isStateDrvTxn(txn, drv)){
Path statePath = drv.stateOutputs.find("state")->second.statepath;
printMsg(lvlTalkative, format("scanning for component and state references inside `%1%'") % statePath);
PathSet state_references = scanForReferences(statePath, allPaths);
PathSet state_stateReferences = scanForStateReferences(statePath, allStatePaths);
all_state_references = mergePathSets(all_state_references, mergePathSets(state_references, state_stateReferences));

View file

@ -507,16 +507,24 @@ void LocalStore::queryStateReferrers(const Path & storePath, PathSet & stateRefe
void setDeriver(const Transaction & txn, const Path & storePath, const Path & deriver)
{
printMsg(lvlError, format("xxxxxxxxxxxxxxxxxxxxxxx"));
assertStorePath(storePath);
printMsg(lvlError, format("Ttttttttttttttttttttttttt"));
if (deriver == "") return;
printMsg(lvlError, format("uuuuuuuuuuuuuuuuuuuuuuuuuuuuu"));
assertStorePath(deriver);
printMsg(lvlError, format("yyyyyyyyyyyyyyyyyyyyyyyyy"));
if (!isRealisablePath(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
if (isStateDrvPathTxn(txn, deriver)){ //Redirect if its a state component
printMsg(lvlError, format("bbbbbbbbbbbbbbb"));
addStateDeriver(txn, storePath, deriver);
}
else{
printMsg(lvlError, format("ccccccccccccccccccc"));
nixDB.setString(txn, dbDerivers, storePath, deriver);
}
}
@ -530,13 +538,19 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path
if (!isRealisablePath(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
printMsg(lvlError, format("dddddddddddddd"));
Derivation drv = derivationFromPath(deriver);
string identifier = drv.stateOutputs.find("state")->second.stateIdentifier;
string user = drv.stateOutputs.find("state")->second.username;
printMsg(lvlError, format("eeeeeeeeeeeeeeeeee"));
PathSet currentDerivers = queryDerivers(txn, storePath, identifier, user);
PathSet updatedDerivers = mergeNewDerivationIntoList(storePath, deriver, currentDerivers, true);
printMsg(lvlError, format("ffffffffffffffffffff"));
Strings data;
for (PathSet::iterator i = updatedDerivers.begin(); i != updatedDerivers.end(); ++i) //Convert Paths to Strings
data.push_back(*i);
@ -597,7 +611,7 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
bool b = nixDB.queryString(txn, dbDerivers, storePath, deriver);
Derivation drv = derivationFromPath(deriver);
if (drv.outputs.size() != 0)
if (isStateDrvTxn(txn, drv))
throw Error(format("This deriver `%1%' is a state deriver, u should use queryDerivers instead of queryDeriver") % deriver);
if (b)
@ -769,18 +783,14 @@ void clearSubstitutes()
static void setHash(const Transaction & txn, const Path & storePath, const Hash & hash, bool stateHash = false)
{
if(stateHash){
nixDB.setString(txn, dbValidStatePaths, storePath, "");
}
else{
nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash));
assert(hash.type == htSHA256);
}
nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash));
assert(hash.type == htSHA256);
}
static void setStateHash(const Transaction & txn, const Path & storePath, const Hash & hash)
static void setStateValid(const Transaction & txn, const Path & statePath, const Path & drvPath)
{
setHash(txn, storePath, hash, true);
printMsg(lvlError, format("setStateValid: '%1%' '%2%'") % statePath % drvPath);
nixDB.setString(txn, dbValidStatePaths, statePath, drvPath);
}
static Hash queryHash(const Transaction & txn, const Path & storePath)
@ -806,6 +816,20 @@ Hash LocalStore::queryPathHash(const Path & path)
return queryHash(noTxn, path);
}
static Path queryStatePathDrv(const Transaction & txn, const Path & statePath)
{
string s;
nixDB.queryString(txn, dbValidStatePaths, statePath, s);
return s;
}
Path LocalStore::queryStatePathDrv(const Path & statePath)
{
if (!isValidStatePath(statePath))
throw Error(format("statepath `%1%' is not valid") % statePath);
return nix::queryStatePathDrv(noTxn, statePath);
}
void registerValidPath(const Transaction & txn,
const Path & path, const Path & statePath, const Hash & hash,
@ -839,19 +863,18 @@ void registerValidPaths(const Transaction & txn, const ValidPathInfos & infos)
setHash(txn, i->path, i->hash);
if (i->statePath != "")
setStateHash(txn, i->statePath, Hash()); //the hash value in the db now becomes empty, but if the key exists, we know that the state path is valid
setStateValid(txn, i->statePath, i->deriver); //if the key exists, we know that the state path is valid, we set the value to the drvPath
setReferences(txn, i->path, i->references, i->stateReferences);
/* Check that all referenced paths are also valid (or about to) become valid). */
//TODO Maybe also check this for stateReferences????
for (PathSet::iterator j = i->references.begin();
j != i->references.end(); ++j)
if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end())
throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
% i->path % *j);
//TODO Also do this for stateReferences????
setDeriver(txn, i->path, i->deriver);
}
}
@ -1489,11 +1512,79 @@ void storePathRequisites(const Path & storePath, const bool includeOutputs, Path
}
}
/*
* Same as storePathRequisites with withState=true, but now only returns the state paths
*/
void storePathStateRequisitesOnlyTxn(const Transaction & txn, const Path & storePath, const bool includeOutputs, PathSet & statePaths)
{
PathSet paths;
storePathRequisites(storePath, includeOutputs, paths, true);
//filter out all non-state paths
for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i){
if (isValidStatePathTxn(txn, *i))
statePaths.insert(*i);
}
}
void LocalStore::storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths)
{
nix::storePathStateRequisitesOnlyTxn(noTxn, storePath, includeOutputs, statePaths);
}
void LocalStore::storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState)
{
return nix::storePathRequisites(storePath, includeOutputs, paths, withState);
}
void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath)
{
//TODO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
}
void getDependenciesAtBuildTime(const Transaction & txn, const Path & drvPath)
{
Derivation drv = derivationFromPath(drvPath);
PathSet allPaths;
PathSet inputPaths;
//TODO THIS IS A DIRECT COPY FROM BUILD.CC WE SHOULD MERGE !!!!!!!!!!!!1!!!!!!!!!!!!!
/* The outputs are referenceable paths. */
for (DerivationOutputs::iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
{
debug(format("building path `%1%'") % i->second.path);
allPaths.insert(i->second.path);
}
/* First, the input derivations. */
for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
{
/* Add the relevant output closures of the input derivation
`*i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(store->isValidPath(i->first));
Derivation inDrv = derivationFromPath(i->first);
for (StringSet::iterator j = i->second.begin(); j != i->second.end(); ++j)
if (inDrv.outputs.find(*j) != inDrv.outputs.end())
computeFSClosure(inDrv.outputs[*j].path, inputPaths, false); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO COPY STATE
else
throw Error(format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") % drvPath % *j % i->first);
}
/* Second, the input sources. */
for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
computeFSClosure(*i, inputPaths, false); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! WE (MAY) ALSO NEED TO COPY STATE
//debug(format("added input paths %1%") % showPaths(inputPaths)); //TODO
allPaths.insert(inputPaths.begin(), inputPaths.end());
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i)
printMsg(lvlError, format("ALLPATHS2: %1%") % *i);
}
/* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
static void upgradeStore07()
{

View file

@ -48,6 +48,8 @@ public:
Substitutes querySubstitutes(const Path & srcPath);
Hash queryPathHash(const Path & path);
Path queryStatePathDrv(const Path & statePath);
void queryReferences(const Path & path, PathSet & references);
@ -96,6 +98,8 @@ public:
void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState);
void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths);
};
@ -205,6 +209,8 @@ bool isStateComponentTxn(const Transaction & txn, const Path & path);
bool isStateDrvPathTxn(const Transaction & txn, const Path & drvPath);
bool isStateDrvTxn(const Transaction & txn, const Derivation & drv);
void convertStatePathsToDerivations(const Transaction & txn, const Path & storePath);
}

View file

@ -17,28 +17,32 @@ Derivation derivationFromPath(const Path & drvPath)
return parseDerivation(t);
}
/*
void computeFSClosure(const Path & storePath,
PathSet & paths, bool flipDirection)
{
if (paths.find(storePath) != paths.end()) return;
paths.insert(storePath);
PathSet references;
if (flipDirection)
store->queryReferrers(storePath, references);
else
store->queryReferences(storePath, references);
for (PathSet::iterator i = references.begin(); i != references.end(); ++i)
computeFSClosure(*i, paths, flipDirection);
}
*/
void computeFSClosure(const Path & path, PathSet & paths, const bool & withState, bool flipDirection)
{
if (paths.find(path) != paths.end()) return;
PathSet allPaths;
computeFSClosureRec(path, allPaths, flipDirection);
//if withState is false, we filter out all state paths
if(withState == false){
for (PathSet::iterator i = allPaths.begin(); i != allPaths.end(); ++i){
if ( ! store->isValidStatePath(*i) ){
paths.insert(*i);
//TODO (OBSOLETE) CHECK TO SEE IF THERE WERE NO /NIX/STATE PATHS THAT ARENT VALID AT THIS POINT, REMOVE THIS IN THE FUTURE
string test = "/nix/state";
if((*i).substr(0, test.size()) == test)
throw Error(format("THIS CANNOT HAPPEN ! computeFSClosure is called before the state path was valid...."));
}
}
}
else{
paths = allPaths;
}
}
void computeFSClosureRec(const Path & path, PathSet & paths, const bool & flipDirection)
{
if (paths.find(path) != paths.end()) return; //takes care of double entries
paths.insert(path);
@ -47,23 +51,18 @@ void computeFSClosure(const Path & path, PathSet & paths, const bool & withState
if (flipDirection){
store->queryReferrers(path, references);
if(withState)
store->queryStateReferrers(path, stateReferences);
store->queryStateReferrers(path, stateReferences);
}
else{
store->queryReferences(path, references);
if(withState)
store->queryStateReferences(path, stateReferences);
store->queryStateReferences(path, stateReferences);
}
PathSet allReferences;
if(withState)
allReferences = mergePathSets(references, stateReferences);
else
allReferences = references;
allReferences = mergePathSets(references, stateReferences);
for (PathSet::iterator i = allReferences.begin(); i != allReferences.end(); ++i)
computeFSClosure(*i, paths, withState, flipDirection);
computeFSClosureRec(*i, paths, flipDirection);
}

View file

@ -18,8 +18,9 @@ Derivation derivationFromPath(const Path & drvPath);
`storePath' is returned; that is, the closures under the
`referrers' relation instead of the `references' relation is
returned. */
void computeFSClosure(const Path & storePath,
PathSet & paths, const bool & withState, bool flipDirection = false);
void computeFSClosure(const Path & storePath, PathSet & paths, const bool & withState, bool flipDirection = false);
void computeFSClosureRec(const Path & path, PathSet & paths, const bool & flipDirection); //private
/* Return the path corresponding to the output identifier `id' in the
given derivation. */

View file

@ -206,6 +206,14 @@ Hash RemoteStore::queryPathHash(const Path & path)
return parseHash(htSHA256, hash);
}
Path RemoteStore::queryStatePathDrv(const Path & statePath)
{
writeInt(wopQueryStatePathDrv, to);
writeString(statePath, to);
processStderr();
Path p = readString(from); //TODO !!!!!!!!!!!!!!!!!!!!!!!!!!! check wheter from is the state path ????
return p;
}
void RemoteStore::queryReferences(const Path & path,
PathSet & references)
@ -449,4 +457,10 @@ void RemoteStore::storePathRequisites(const Path & storePath, const bool include
}
//TODO
void RemoteStore::storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths)
{
}
}

View file

@ -36,6 +36,8 @@ public:
bool hasSubstitutes(const Path & path);
Hash queryPathHash(const Path & path);
Path queryStatePathDrv(const Path & statePath);
void queryReferences(const Path & path, PathSet & references);
@ -84,6 +86,7 @@ public:
void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState);
void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths);
private:
AutoCloseFD fdSocket;

View file

@ -115,9 +115,7 @@ void checkStatePath(const Derivation & drv)
Error(format("The statepath from the Derivation does not match the recalculated statepath, are u trying to spoof the statepath?"));
}
Path makeStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier)
void calculateStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier, Path & rootPath, Path & fullPath)
{
//This is a little trick: we could use the same hash as the statepath, but we change it so the repository also gets a unique scannable hash
Hash hash = hashString(htSHA256, statePath);
@ -127,7 +125,7 @@ Path makeStateReposPath(const string & type, const Path statePath, const string
throw Error(format("Cannot create a repository for a subfolder without a name"));
string hash_subfolder = type + ":sha256:" + printHash(hash) + ":" + subfolder;
string subfolder_ = printHash32(compressHash(hashString(htSHA256, hash_subfolder), 20)) + "-" + subfolder;
string subfolder2 = printHash32(compressHash(hashString(htSHA256, hash_subfolder), 20)) + "-" + subfolder;
string suffix_stateIdentifier = stateIdentifier;
if(suffix_stateIdentifier != "")
@ -140,9 +138,27 @@ Path makeStateReposPath(const string & type, const Path statePath, const string
checkStoreName(suffix);
checkStoreName(stateIdentifier);
return nixStoreStateRepos + "/"
rootPath = nixStoreStateRepos + "/"
+ printHash32(compressHash(hashString(htSHA256, s), 20))
+ "-" + suffix + suffix_stateIdentifier + "/" + subfolder_;
+ "-" + suffix + suffix_stateIdentifier;
fullPath = rootPath + "/" + subfolder2;
}
Path getStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier)
{
Path fullPath;
Path rootPath;
calculateStateReposPath(type, statePath, subfolder, suffix, stateIdentifier, rootPath, fullPath);
return fullPath;
}
Path getStateReposRootPath(const string & type, const Path statePath, const string & suffix, const string & stateIdentifier)
{
Path fullPath;
Path rootPath;
calculateStateReposPath(type, statePath, "/", suffix, stateIdentifier, rootPath, fullPath);
return rootPath;
}
Path makeFixedOutputPath(bool recursive,

View file

@ -75,6 +75,9 @@ public:
/* Queries the hash of a valid path. */
virtual Hash queryPathHash(const Path & path) = 0;
/* Queries the derivation Path of a valid state path. */
virtual Path queryStatePathDrv(const Path & statePath) = 0;
/* Queries the set of outgoing FS references for a store path.
The result is not cleared. */
virtual void queryReferences(const Path & path,
@ -212,7 +215,8 @@ public:
virtual bool isStateDrv(const Derivation & drv) = 0;
virtual void storePathRequisites(const Path & storePath, const bool includeOutputs, PathSet & paths, const bool & withState) = 0;
virtual void storePathStateRequisitesOnly(const Path & storePath, const bool includeOutputs, PathSet & statePaths) = 0;
};
@ -245,9 +249,14 @@ Path makeStatePath(const string & componentHash, const string & suffix, const st
/* TODO ... */
void checkStatePath(const Derivation & drv);
/* Constructs a unique store state repos path name. */
Path makeStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier);
/* Calculates a unique store state repos path and also the root path */
void calculateStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier, Path & rootPath, Path & fullPath);
/* Returns the full repository path */
Path getStateReposPath(const string & type, const Path statePath, const string subfolder, const string & suffix, const string & stateIdentifier);
/* Returns the root path containing the repository's */
Path getStateReposRootPath(const string & type, const Path statePath, const string & suffix, const string & stateIdentifier);
/* This is the preparatory part of addToStore() and addToStoreFixed();
it computes the store path to which srcPath is to be copied.

View file

@ -39,6 +39,10 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De
string svnadminbin = nixSVNPath + "/svnadmin";
PathSet intervalPaths;
//Make sure the 'root' path which holds the repositorys exists, so svn doenst complain.
string repos_root_path = getStateReposRootPath("stateOutput:staterepospath", stateDir, drvName, stateIdentifier);
executeAndPrintShellCommand("mkdir -p " + repos_root_path, "mkdir");
//TODO check if we can create state and staterepos dirs
@ -56,13 +60,13 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De
}
//Create a repository for this state location
string repos = makeStateReposPath("stateOutput:staterepospath", stateDir, thisdir, drvName, stateIdentifier);
executeAndPrintShellCommand("mkdir -p " + repos, "mkdir");
string repos = getStateReposPath("stateOutput:staterepospath", stateDir, thisdir, drvName, stateIdentifier);
if(IsDirectory(repos))
printMsg(lvlTalkative, format("Repos %1% already exists, so we use that repository") % repos);
else
executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit than ??
executeAndPrintShellCommand(svnadminbin + " create " + repos, "svnadmin"); //TODO create as nixbld.nixbld chmod 700... can you still commit then ??
if(d.type == "interval"){
intervalPaths.insert(statePath);
@ -70,12 +74,13 @@ void createStateDirs(const DerivationStateOutputDirs & stateOutputDirs, const De
printMsg(lvlTalkative, format("Adding state subdir: %1% to %2% from repository %3%") % thisdir % fullstatedir % repos);
if(IsDirectory(fullstatedir + "/.svn/")){
string fullstatedir_svn = fullstatedir + "/.svn/";
if( ! IsDirectory(fullstatedir_svn) ){
string checkoutcommand = svnbin + " checkout file://" + repos + " " + fullstatedir;
executeAndPrintShellCommand(checkoutcommand, "svn"); //TODO checkout as user
}
else
printMsg(lvlTalkative, format("Statedir %1% already exists, so dont check out its repository again") % fullstatedir);
printMsg(lvlTalkative, format("Statedir %1% already exists, so dont check out its repository again") % fullstatedir_svn);
}
//Initialize the counters for the statePaths that have an interval to 0

View file

@ -17,6 +17,7 @@ typedef enum {
wopQuerySubstitutes,
wopHasSubstitutes,
wopQueryPathHash,
wopQueryStatePathDrv,
wopQueryReferences,
wopQueryStateReferences,
wopQueryReferrers,