* We now actually do hash rewriting. Builders build temporary store

paths (e.g., `/nix/store/...random-hash...-aterm'), which are
  subsequently rewritten to actual content-addressable store paths
  (i.e., the hash part of the store path equals the hash of the
  contents).

  A complication is that the temporary output paths have to be passed
  to the builder (e.g., in $out).  Likewise, other environment
  variables and command-line arguments cannot contain fixed store
  paths because their names are no longer known in advance.
  
  Therefore, we now put placeholder store paths in environment
  variables and command-line arguments, which we *rewrite* to the
  actual paths prior to running the builder.

  TODO: maintain the mapping of derivation placeholder outputs
  ("output path equivalence classes") to actual output paths in the
  database.  Right now the first build succeeds and all its
  dependencies fail because they cannot find the output of the first.

  TODO: locking is no longer an issue with random temporary paths, but
  at the cost of having no blocking if we build the same thing twice
  in parallel.  Maybe the "random" path should actually be a hash of
  the placeholder and the name of the user who started the build.

src/libexpr/primops.cc

@@ -71,8 +71,8 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
         {
             return hashString(htSHA256, "fixed:out:"
                 + i->second.hashAlgo + ":"
-                + i->second.hash + ":"
+                + i->second.hash /* !!! + ":"
-                + i->second.path);
+                                    + i->second.path */);
         }
     }
 
@@ -319,8 +319,10 @@ static Expr primDerivationStrict(EvalState & state, const ATermVector & args)
         
     /* Use the masked derivation expression to compute the output
        path. */
-    Path outPath = makeStorePath("output:out",
+    /* XXX */
-        hashDerivationModulo(state, drv), drvName);
+    Path outPath;
+    PathHash outPathHash;
+    makeStorePath(hashDerivationModulo(state, drv), drvName, outPath, outPathHash);
 
     /* Construct the final derivation store expression. */
     drv.env["out"] = outPath;

src/libstore/Makefile.am

@@ -16,10 +16,3 @@ derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
 	$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
 
 derivations.cc store.cc: derivations-ast.hh
-
-
-bin_PROGRAMS = testprog
-
-testprog_SOURCES = store-new.cc
-testprog_LDADD = ../libutil/libutil.a \
- ../boost/format/libformat.a ${bdb_lib} ${aterm_lib}

src/libstore/build.cc

@@ -314,6 +314,16 @@ private:
     
     /* The remainder is state held during the build. */
 
+    /* The map of output equivalence classes to temporary output
+       paths. */
+    typedef map<OutputEqClass, Path> OutputMap;
+    OutputMap tmpOutputs;
+
+    /* The hash rewrite map that rewrites output equivalences occuring
+       in the command-line arguments and environment variables to the
+       actual paths to be used. */
+    HashRewrites rewrites;
+
     /* Locks on the output paths. */
     PathLocks outputLocks;
 
@@ -463,6 +473,7 @@ void DerivationGoal::haveStoreExpr()
     /* Get the derivation. */
     drv = derivationFromPath(drvPath);
 
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
         addTempRoot(i->second.path);
@@ -485,6 +496,7 @@ void DerivationGoal::haveStoreExpr()
            substitutes. */
         if (querySubstitutes(noTxn, *i).size() > 0)
             addWaitee(worker.makeSubstitutionGoal(*i));
+#endif    
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
@@ -502,10 +514,12 @@ void DerivationGoal::outputsSubstituted()
 
     nrFailed = 0;
 
+#if 0    
     if (checkPathValidity(false).size() == 0) {
         amDone(true);
         return;
     }
+#endif    
 
     /* Otherwise, at least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
@@ -647,6 +661,7 @@ void DerivationGoal::buildDone()
 }
 
 
+#if 0
 static string readLine(int fd)
 {
     string s;
@@ -686,16 +701,20 @@ static void drain(int fd)
         else writeFull(STDERR_FILENO, buffer, rd);
     }
 }
+#endif
 
 
+#if 0
 PathSet outputPaths(const DerivationOutputs & outputs)
 {
     PathSet paths;
+    /* XXX */
     for (DerivationOutputs::const_iterator i = outputs.begin();
          i != outputs.end(); ++i)
         paths.insert(i->second.path);
     return paths;
 }
+#endif
 
 
 string showPaths(const PathSet & paths)
@@ -713,6 +732,9 @@ string showPaths(const PathSet & paths)
 
 DerivationGoal::HookReply DerivationGoal::tryBuildHook()
 {
+    return rpDecline;
+    
+#if 0    
     Path buildHook = getEnv("NIX_BUILD_HOOK");
     if (buildHook == "") return rpDecline;
     buildHook = absPath(buildHook);
@@ -861,6 +883,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     }
 
     else throw Error(format("bad hook reply `%1%'") % reply);
+#endif
 }
 
 
@@ -881,11 +904,27 @@ void DerivationGoal::terminateBuildHook()
 
 bool DerivationGoal::prepareBuild()
 {
+    for (DerivationOutputs::iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+    {
+        Path tmpPath = makeRandomStorePath(namePartOf(i->second.eqClass));
+        printMsg(lvlError, format("mapping output id `%1%', class `%2%' to `%3%'")
+            % i->first % i->second.eqClass % tmpPath);
+        assert(i->second.eqClass.size() == tmpPath.size());
+        rewrites[hashPartOf(i->second.eqClass)] = hashPartOf(tmpPath);
+        tmpOutputs[i->second.eqClass] = tmpPath;
+    }
+    
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes. */
     /* !!! BUG: this could block, which is not allowed. */
+#if 0
+    /* !!! acquire lock on the derivation or something? or on a
+       pseudo-path representing the output equivalence class? */
     outputLocks.lockPaths(outputPaths(drv.outputs));
+#endif
 
+#if 0    
     /* Now check again whether the outputs are valid.  This is because
        another process may have started building in parallel.  After
        it has finished and released the locks, we can (and should)
@@ -907,10 +946,12 @@ bool DerivationGoal::prepareBuild()
             format("derivation `%1%' is blocked by its output paths")
             % drvPath);
     }
+#endif    
 
     /* Gather information necessary for computing the closure and/or
        running the build hook. */
 
+#if 0    
     /* The outputs are referenceable paths. */
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
@@ -948,6 +989,7 @@ bool DerivationGoal::prepareBuild()
     debug(format("added input paths %1%") % showPaths(inputPaths));
 
     allPaths.insert(inputPaths.begin(), inputPaths.end());
+#endif    
 
     return true;
 }
@@ -956,7 +998,7 @@ bool DerivationGoal::prepareBuild()
 void DerivationGoal::startBuilder()
 {
     startNest(nest, lvlInfo,
-        format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
+        format("building path(s) XXX") /* % showPaths(outputPaths(drv.outputs)) */)
     
     /* Right platform? */
     if (drv.platform != thisSystem)
@@ -966,6 +1008,7 @@ void DerivationGoal::startBuilder()
 
     /* If any of the outputs already exist but are not registered,
        delete them. */
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin(); 
          i != drv.outputs.end(); ++i)
     {
@@ -977,6 +1020,7 @@ void DerivationGoal::startBuilder()
             deletePath(path);
         }
     }
+#endif
 
     /* Construct the environment passed to the builder. */
     typedef map<string, string> Environment;
@@ -1004,7 +1048,7 @@ void DerivationGoal::startBuilder()
     /* Add all bindings specified in the derivation. */
     for (StringPairs::iterator i = drv.env.begin();
          i != drv.env.end(); ++i)
-        env[i->first] = i->second;
+        env[i->first] = rewriteHashes(i->second, rewrites);
 
     /* Create a temporary directory where the build will take
        place. */
@@ -1061,7 +1105,8 @@ void DerivationGoal::startBuilder()
             Strings envStrs;
             for (Environment::const_iterator i = env.begin();
                  i != env.end(); ++i)
-                envStrs.push_back(i->first + "=" + i->second);
+                envStrs.push_back(i->first + "=" +
+                    rewriteHashes(i->second, rewrites));
             const char * * envArr = strings2CharPtrs(envStrs);
 
             /* Execute the program.  This should not return. */
@@ -1090,6 +1135,18 @@ void DerivationGoal::computeClosure()
     map<Path, PathSet> allReferences;
     map<Path, Hash> contentHashes;
 
+    for (OutputMap::iterator i = tmpOutputs.begin();
+         i != tmpOutputs.end(); ++i)
+    {
+        /* Rewrite each output to a name matching its content hash.
+           I.e., enforce the hash invariant: the hash part of a store
+           path matches the contents at that path. */
+        Path finalPath = addToStore(i->second, hashPartOf(i->second),
+            namePartOf(i->second));
+        printMsg(lvlError, format("produced final path `%1%'") % finalPath);
+    }
+    
+#if 0    
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
@@ -1175,7 +1232,9 @@ void DerivationGoal::computeClosure()
            if we could combine this with filterReferences(). */
         contentHashes[path] = hashPath(htSHA256, path);
     }
+#endif
 
+#if 0    
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  This is wrapped in one
        database transaction to ensure that if we crash, either
@@ -1198,6 +1257,7 @@ void DerivationGoal::computeClosure()
             drvPath);
     }
     txn.commit();
+#endif
     
     /* It is now safe to delete the lock files, since all future
        lockers will see that the output paths are valid; they will not
@@ -1280,6 +1340,7 @@ void DerivationGoal::writeLog(int fd,
 
 PathSet DerivationGoal::checkPathValidity(bool returnValid)
 {
+#if 0    
     PathSet result;
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
@@ -1289,6 +1350,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
             if (!returnValid) result.insert(i->second.path);
         }
     return result;
+#endif
 }
 
 
@@ -1302,11 +1364,13 @@ private:
     /* The store path that should be realised through a substitute. */
     Path storePath;
 
+#if 0    
     /* The remaining substitutes for this path. */
     Substitutes subs;
 
     /* The current substitute. */
     Substitute sub;
+#endif    
 
     /* Outgoing references for this path. */
     PathSet references;
@@ -1375,10 +1439,12 @@ void SubstitutionGoal::init()
         return;
     }
 
+#if 0
     /* Read the substitutes. */
     subs = querySubstitutes(noTxn, storePath);
+#endif
 
-    /* To maintain the closure invairant, we first have to realise the
+    /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
     queryReferences(noTxn, storePath, references);
 
@@ -1414,7 +1480,7 @@ void SubstitutionGoal::tryNext()
 {
     trace("trying next substitute");
 
-    if (subs.size() == 0) {
+    if (true /* !!! subs.size() == 0 */) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
         printMsg(lvlError,
@@ -1423,8 +1489,10 @@ void SubstitutionGoal::tryNext()
         amDone(false);
         return;
     }
+#if 0    
     sub = subs.front();
     subs.pop_front();
+#endif
 
     /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
@@ -1434,6 +1502,7 @@ void SubstitutionGoal::tryNext()
 
 void SubstitutionGoal::tryToRun()
 {
+#if 0
     trace("trying to run");
 
     /* Make sure that we are allowed to start a build. */
@@ -1505,11 +1574,13 @@ void SubstitutionGoal::tryToRun()
         pid, logPipe.readSide, true);
 
     state = &SubstitutionGoal::finished;
+#endif    
 }
 
 
 void SubstitutionGoal::finished()
 {
+#if 0    
     trace("substitute finished");
 
     /* Since we got an EOF on the logger pipe, the substitute is
@@ -1566,6 +1637,7 @@ void SubstitutionGoal::finished()
         format("substitution of path `%1%' succeeded") % storePath);
 
     amDone();
+#endif
 }
 
 

src/libstore/derivations.cc

@@ -62,12 +62,12 @@ Derivation parseDerivation(ATerm t)
         throwBadDrv(t);
 
     for (ATermIterator i(outs); i; ++i) {
-        ATerm id, path, hashAlgo, hash;
+        ATerm id, eqClass, hashAlgo, hash;
-        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
+        if (!matchDerivationOutput(*i, id, eqClass, hashAlgo, hash))
             throwBadDrv(t);
         DerivationOutput out;
-        out.path = aterm2String(path);
+        out.eqClass = aterm2String(eqClass);
-        checkPath(out.path);
+        // !!!        checkPath(out.path);
         out.hashAlgo = aterm2String(hashAlgo);
         out.hash = aterm2String(hash);
         drv.outputs[aterm2String(id)] = out;
@@ -125,7 +125,7 @@ ATerm unparseDerivation(const Derivation & drv)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
-                toATerm(i->second.path),
+                toATerm(i->second.eqClass),
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 

src/libstore/derivations.hh

@@ -13,15 +13,15 @@ const string drvExtension = ".drv";
 
 struct DerivationOutput
 {
-    Path path;
+    OutputEqClass eqClass;
     string hashAlgo; /* hash used for expected hash computation */
     string hash; /* expected hash, may be null */
     DerivationOutput()
     {
     }
-    DerivationOutput(Path path, string hashAlgo, string hash)
+    DerivationOutput(OutputEqClass eqClass, string hashAlgo, string hash)
     {
-        this->path = path;
+        this->eqClass = eqClass;
         this->hashAlgo = hashAlgo;
         this->hash = hash;
     }

src/libstore/gc.cc

@@ -338,12 +338,14 @@ void collectGarbage(GCAction action, PathSet & result)
         for (PathSet::iterator i = livePaths.begin();
              i != livePaths.end(); ++i)
         {
+#if 0            
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
             Path deriver = queryDeriver(noTxn, *i);
             if (deriver != "" && isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
+#endif            
         }
     }
 
@@ -353,10 +355,13 @@ void collectGarbage(GCAction action, PathSet & result)
              i != livePaths.end(); ++i)
             if (isDerivation(*i)) {
                 Derivation drv = derivationFromPath(*i);
+                assert(0);
+#if 0                
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
                     if (isValidPath(j->second.path))
                         computeFSClosure(j->second.path, livePaths);
+#endif
             }
     }
 

src/libstore/misc.cc

@@ -29,10 +29,13 @@ void computeFSClosure(const Path & storePath,
 }
 
 
- Path findOutput(const Derivation & drv, string id)
+Path findOutput(const Derivation & drv, string id)
 {
+    assert(0);
+#if 0    
     for (DerivationOutputs::const_iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
         if (i->first == id) return i->second.path;
     throw Error(format("derivation has no output `%1%'") % id);
+#endif    
 }

src/libstore/store-new.cc

@@ -1,173 +0,0 @@
-#include "store-new.hh"
-
-#include "util.hh"
-#include "archive.hh"
-
-
-const unsigned int pathHashLen = 32; /* characters */
-const string nullPathHashRef(pathHashLen, 0);
-
-
-PathHash::PathHash()
-{
-    rep = nullPathHashRef;
-}
-
-
-PathHash::PathHash(const Hash & h)
-{
-    assert(h.type == htSHA256);
-    rep = printHash32(compressHash(h, 20));
-}
-
-
-string PathHash::toString() const
-{
-    return rep;
-}
-
-
-bool PathHash::isNull() const
-{
-    return rep == nullPathHashRef;
-}
-
-
-bool PathHash::operator ==(const PathHash & hash2) const
-{
-    return rep == hash2.rep;
-}
-
-
-bool PathHash::operator <(const PathHash & hash2) const
-{
-    return rep < hash2.rep;
-}
-
-
-PathHash generateRandomPathHash()
-{
-    Hash hash(htSHA256);
-    for (unsigned int i = 0; i < hash.hashSize; ++i)
-        hash.hash[i] = rand() % 256; // !!! improve
-    return PathHash(hash);
-}
-
-
-struct CopySink : DumpSink
-{
-    string s;
-    virtual void operator () (const unsigned char * data, unsigned int len)
-    {
-        s.append((const char *) data, len);
-    }
-};
-
-
-struct CopySource : RestoreSource
-{
-    string & s;
-    unsigned int pos;
-    CopySource(string & _s) : s(_s), pos(0) { }
-    virtual void operator () (unsigned char * data, unsigned int len)
-    {
-        s.copy((char *) data, len, pos);
-        pos += len;
-        assert(pos <= s.size());
-    }
-};
-
-
-static string rewriteHashes(string s, const HashRewrites & rewrites,
-    vector<int> & positions)
-{
-    for (HashRewrites::const_iterator i = rewrites.begin();
-         i != rewrites.end(); ++i)
-    {
-        string from = i->first.toString(), to = i->second.toString();
-
-        assert(from.size() == to.size());
-
-        unsigned int j = 0;
-        while ((j = s.find(from, j)) != string::npos) {
-            debug(format("rewriting @ %1%") % j);
-            positions.push_back(j);
-            s.replace(j, to.size(), to);
-        }
-    }
-
-    return s;
-}
-
-
-PathHash hashModulo(string s, const PathHash & modulus)
-{
-    vector<int> positions;
-    
-    if (!modulus.isNull()) {
-        /* Zero out occurences of `modulus'. */
-        HashRewrites rewrites;
-        rewrites[modulus] = PathHash(); /* = null hash */
-        s = rewriteHashes(s, rewrites, positions);
-    }
-
-    string positionPrefix;
-    
-    for (vector<int>::iterator i = positions.begin();
-         i != positions.end(); ++i)
-        positionPrefix += (format("|%1%") % *i).str();
-
-    positionPrefix += "||";
-
-    debug(format("positions %1%") % positionPrefix);
-    
-    return PathHash(hashString(htSHA256, positionPrefix + s));
-}
-
-
-Path addToStore(const Path & srcPath, const PathHash & selfHash)
-{
-    debug(format("adding %1%") % srcPath);
-
-    CopySink sink;
-    dumpPath(srcPath, sink);
-
-    PathHash newHash = hashModulo(sink.s, selfHash);
-
-    debug(format("newHash %1%") % newHash.toString());
-
-    if (!selfHash.isNull()) {
-        HashRewrites rewrites;
-        rewrites[selfHash] = newHash;
-        vector<int> positions;
-        sink.s = rewriteHashes(sink.s, rewrites, positions);
-        PathHash newHash2 = hashModulo(sink.s, newHash);
-        assert(newHash2 == newHash);
-        debug(format("newHash2 %1%") % newHash2.toString());
-    }
-
-    Path path = "./out/" + newHash.toString() + "-" + baseNameOf(srcPath);
-
-    CopySource source(sink.s);
-    restorePath(path, source);
-
-    return path;
-}
-
-
-int main(int argc, char * * argv)
-{
-    verbosity = (Verbosity) ((int) 10);
-
-    srand(time(0));
-
-    debug(format("%1%") % generateRandomPathHash().toString());
-    debug(format("%1%") % generateRandomPathHash().toString());
-    debug(format("%1%") % generateRandomPathHash().toString());
-    
-    Path p = addToStore("./foo", PathHash(parseHash32(htSHA256, "8myr6ajc52b5sky7iplgz8jv703ljc0q")));
-
-    cout << p << endl;
-    
-    return 0;
-}

src/libstore/store-new.hh

@@ -1,38 +0,0 @@
-#ifndef __STORE_H
-#define __STORE_H
-
-#include <string>
-#include <map>
-
-#include "hash.hh"
-#include "db.hh"
-
-using namespace std;
-
-
-struct PathHash
-{
-private:
-    string rep;
-public:
-    PathHash();
-    PathHash(const Hash & h);
-    string toString() const;
-    bool PathHash::isNull() const;
-    bool operator ==(const PathHash & hash2) const;
-    bool operator <(const PathHash & hash2) const;
-};
-
-
-/* Add the contents of the specified path to the Nix store.  Any
-   occurence of the representation of `selfHash' (if not empty) is
-   rewritten to the hash of the new store path. */
-Path addToStore(const Path & srcPath, const PathHash & selfHash);
-
-
-/* Rewrite a set of hashes in the given path. */
-typedef map<PathHash, PathHash> HashRewrites;
-//Path rewriteHashes(const Path & srcPath, HashRewrites rewrites);
-
-
-#endif /* !__STORE_H */

src/libstore/store.cc

@@ -39,6 +39,19 @@ static TableId dbReferences = 0;
    This table is just the reverse mapping of dbReferences. */
 static TableId dbReferers = 0;
 
+/* dbEquivalences :: OutputEqClass -> [(TrustId, Path)]
+
+   Lists the output paths that have been produced for each extension
+   class; i.e., the extension of an extension class. */
+static TableId dbEquivalences = 0;
+
+/* dbEquivalenceClass :: Path -> OutputEqClass
+
+   Lists for each output path the extension class that it is in. */
+static TableId dbEquivalenceClass = 0;
+
+
+#if 0
 /* dbSubstitutes :: Path -> [[Path]]
 
    Each pair $(p, subs)$ tells Nix that it can use any of the
@@ -61,13 +74,16 @@ static TableId dbSubstitutes = 0;
    only be multiple such paths for fixed-output derivations (i.e.,
    derivations specifying an expected hash). */
 static TableId dbDerivers = 0;
+#endif
 
 
+#if 0
 bool Substitute::operator == (const Substitute & sub) const
 {
     return program == sub.program
         && args == sub.args;
 }
+#endif
 
 
 static void upgradeStore();
@@ -87,8 +103,12 @@ void openDB()
     dbValidPaths = nixDB.openTable("validpaths");
     dbReferences = nixDB.openTable("references");
     dbReferers = nixDB.openTable("referers");
+#if 0    
     dbSubstitutes = nixDB.openTable("substitutes");
     dbDerivers = nixDB.openTable("derivers");
+#endif
+    dbEquivalences = nixDB.openTable("equivalences");
+    dbEquivalenceClass = nixDB.openTable("equivalence-class");
 
     int curSchema = 0;
     Path schemaFN = nixDBPath + "/schema";
@@ -121,6 +141,60 @@ void createStoreTransaction(Transaction & txn)
 }
 
 
+
+/* Path hashes. */
+
+const unsigned int pathHashLen = 32; /* characters */
+const string nullPathHashRef(pathHashLen, 0);
+
+
+PathHash::PathHash()
+{
+    rep = nullPathHashRef;
+}
+
+
+PathHash::PathHash(const Hash & h)
+{
+    assert(h.type == htSHA256);
+    rep = printHash32(compressHash(h, 20));
+}
+
+
+PathHash::PathHash(const string & h)
+{
+    /* !!! hacky; check whether this is a valid 160 bit hash */
+    assert(h.size() == pathHashLen);
+    parseHash32(htSHA1, h); 
+    rep = h;
+}
+
+
+string PathHash::toString() const
+{
+    return rep;
+}
+
+
+bool PathHash::isNull() const
+{
+    return rep == nullPathHashRef;
+}
+
+
+bool PathHash::operator ==(const PathHash & hash2) const
+{
+    return rep == hash2.rep;
+}
+
+
+bool PathHash::operator <(const PathHash & hash2) const
+{
+    return rep < hash2.rep;
+}
+
+
+
 /* Path copying. */
 
 struct CopySink : DumpSink
@@ -167,12 +241,14 @@ void copyPath(const Path & src, const Path & dst)
 }
 
 
+
 bool isInStore(const Path & path)
 {
     return path[0] == '/'
         && string(path, 0, nixStore.size()) == nixStore
         && path.size() >= nixStore.size() + 2
-        && path[nixStore.size()] == '/';
+        && path[nixStore.size()] == '/'
+        && path[nixStore.size() + 1 + pathHashLen] == '-';
 }
 
 
@@ -202,6 +278,20 @@ Path toStorePath(const Path & path)
 }
 
 
+PathHash hashPartOf(const Path & path)
+{
+    assertStorePath(path);
+    return PathHash(string(path, nixStore.size() + 1, pathHashLen));
+}
+
+
+string namePartOf(const Path & path)
+{
+    assertStorePath(path);
+    return string(path, nixStore.size() + 1 + pathHashLen + 1);
+}
+
+
 void checkStoreName(const string & name)
 {
     string validChars = "+-._?=";
@@ -275,14 +365,16 @@ bool isValidPath(const Path & path)
 }
 
 
+#if 0
 static Substitutes readSubstitutes(const Transaction & txn,
     const Path & srcPath);
+#endif
 
 
 static bool isRealisablePath(const Transaction & txn, const Path & path)
 {
     return isValidPathTxn(txn, path)
-        || readSubstitutes(txn, path).size() > 0;
+        /* !!! || readSubstitutes(txn, path).size() > 0 */;
 }
 
 
@@ -356,6 +448,14 @@ void queryReferers(const Transaction & txn,
 }
 
 
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members)
+{
+    assert(0);
+}
+
+
+#if 0
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver)
 {
@@ -378,8 +478,10 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
     else
         return "";
 }
+#endif
 
 
+#if 0
 const int substituteVersion = 2;
 
 
@@ -488,6 +590,7 @@ void clearSubstitutes()
     
     txn.commit();
 }
+#endif
 
 
 static void setHash(const Transaction & txn, const Path & storePath,
@@ -563,7 +666,9 @@ void registerValidPaths(const Transaction & txn,
                 throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
                     % i->path % *j);
 
+#if 0
         setDeriver(txn, i->path, i->deriver);
+#endif
     }
 }
 
@@ -578,30 +683,40 @@ static void invalidatePath(Transaction & txn, const Path & path)
        inverse `referers' entries, and the `derivers' entry; but only
        if there are no substitutes for this path.  This maintains the
        cleanup invariant. */
-    if (querySubstitutes(txn, path).size() == 0) {
+    if (1 /*querySubstitutes(txn, path).size() == 0 !!! */) {
         setReferences(txn, path, PathSet());
-        nixDB.delPair(txn, dbDerivers, path);
+        // !!!        nixDB.delPair(txn, dbDerivers, path);
     }
     
     nixDB.delPair(txn, dbValidPaths, path);
 }
 
 
-Path makeStorePath(const string & type,
+void makeStorePath(const Hash & contentHash, const string & suffix,
-    const Hash & hash, const string & suffix)
+    Path & path, PathHash & pathHash)
 {
-    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":sha256:" + printHash(hash) + ":"
-        + nixStore + ":" + suffix;
-
     checkStoreName(suffix);
 
-    return nixStore + "/"
+    /* e.g., "sha256:1abc...:/nix/store:foo.tar.gz" */
-        + printHash32(compressHash(hashString(htSHA256, s), 20))
+    string s = "sha256:" + printHash(contentHash) + ":"
-        + "-" + suffix;
+        + nixStore + ":" + suffix;
+
+    pathHash = PathHash(hashString(htSHA256, s));
+    
+    path = nixStore + "/" + pathHash.toString() + "-" + suffix;
 }
 
 
+Path makeRandomStorePath(const string & suffix)
+{
+    Hash hash(htSHA256);
+    for (unsigned int i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = rand() % 256; // !!! improve
+    return nixStore + "/" + PathHash(hash).toString() + "-" + suffix;
+}
+
+
+#if 0
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name)
 {
@@ -612,38 +727,89 @@ Path makeFixedOutputPath(bool recursive,
         + "");
     return makeStorePath("output:out", h, name);
 }
+#endif
 
 
-static Path _addToStore(bool fixed, bool recursive,
+typedef map<PathHash, PathHash> HashRewrites;
-    string hashAlgo, const Path & _srcPath)
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions)
 {
-    Path srcPath(absPath(_srcPath));
+    for (HashRewrites::const_iterator i = rewrites.begin();
-    debug(format("adding `%1%' to the store") % srcPath);
+         i != rewrites.end(); ++i)
-
-    Hash h(htSHA256);
     {
-        SwitchToOriginalUser sw;
+        string from = i->first.toString(), to = i->second.toString();
-        h = hashPath(htSHA256, srcPath);
+
+        assert(from.size() == to.size());
+
+        unsigned int j = 0;
+        while ((j = s.find(from, j)) != string::npos) {
+            debug(format("rewriting @ %1%") % j);
+            positions.push_back(j);
+            s.replace(j, to.size(), to);
+        }
     }
 
-    string baseName = baseNameOf(srcPath);
+    return s;
+}
 
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites)
+{
+    vector<int> dummy;
+    return rewriteHashes(s, rewrites, dummy);
+}
+
+
+static Hash hashModulo(string s, const PathHash & modulus)
+{
+    vector<int> positions;
+    
+    if (!modulus.isNull()) {
+        /* Zero out occurences of `modulus'. */
+        HashRewrites rewrites;
+        rewrites[modulus] = PathHash(); /* = null hash */
+        s = rewriteHashes(s, rewrites, positions);
+    }
+
+    string positionPrefix;
+    
+    for (vector<int>::iterator i = positions.begin();
+         i != positions.end(); ++i)
+        positionPrefix += (format("|%1%") % *i).str();
+
+    positionPrefix += "||";
+
+    debug(format("positions %1%") % positionPrefix);
+    
+    return hashString(htSHA256, positionPrefix + s);
+}
+
+
+static Path _addToStore(const string & suffix, string dump,
+    const PathHash & selfHash, const PathSet & references)
+{
+    /* Hash the contents, modulo the previous hash reference (if it
+       had one). */
+    Hash contentHash = hashModulo(dump, selfHash);
+
+    /* Construct the new store path. */ 
     Path dstPath;
+    PathHash pathHash;
+    makeStorePath(contentHash, suffix, dstPath, pathHash);
 
-    if (fixed) {
+    /* If the contents had a previous hash reference, rewrite those
-
+       references to the new hash. */
-        HashType ht(parseHashType(hashAlgo));
+    if (!selfHash.isNull()) {
-        Hash h2(ht);
+        HashRewrites rewrites;
-        {
+        rewrites[selfHash] = pathHash;
-            SwitchToOriginalUser sw;
+        vector<int> positions;
-            h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath);
+        dump = rewriteHashes(dump, rewrites, positions);
+        /* !!! debug code, remove */
+        PathHash contentHash2 = hashModulo(dump, pathHash);
+        assert(contentHash2 == contentHash);
     }
 
-        dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
-    }
-        
-    else dstPath = makeStorePath("source", h, baseName);
-
     if (!readOnlyMode) addTempRoot(dstPath);
 
     if (!readOnlyMode && !isValidPath(dstPath)) { 
@@ -659,17 +825,13 @@ static Path _addToStore(bool fixed, bool recursive,
 
             if (pathExists(dstPath)) deletePath(dstPath);
 
-            copyPath(srcPath, dstPath);
+            CopySource source(dump);
-
+            restorePath(dstPath, source);
-            Hash h2 = hashPath(htSHA256, dstPath);
-            if (h != h2)
-                throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
-                    % srcPath % dstPath % printHash(h) % printHash(h2));
 
             canonicalisePathMetaData(dstPath);
             
             Transaction txn(nixDB);
-            registerValidPath(txn, dstPath, h, PathSet(), "");
+            registerValidPath(txn, dstPath, contentHash, references, "");
             txn.commit();
         }
 
@@ -680,51 +842,38 @@ static Path _addToStore(bool fixed, bool recursive,
 }
 
 
-Path addToStore(const Path & srcPath)
+Path addToStore(const Path & _srcPath, const PathHash & selfHash,
+    const string & suffix)
 {
-    return _addToStore(false, false, "", srcPath);
+    Path srcPath(absPath(_srcPath));
+    debug(format("adding `%1%' to the store") % srcPath);
+
+    CopySink sink;
+    {
+        SwitchToOriginalUser sw;
+        dumpPath(srcPath, sink);
+    }
+
+    return _addToStore(suffix == "" ? baseNameOf(srcPath) : suffix,
+        sink.s, selfHash, PathSet());
 }
 
 
+#if 0
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath)
 {
     return _addToStore(true, recursive, hashAlgo, srcPath);
 }
+#endif
 
 
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references)
 {
-    Hash hash = hashString(htSHA256, s);
+    CopySink sink;
+    makeSingletonArchive(s, sink);
     
-    Path dstPath = makeStorePath("text", hash, suffix);
+    return _addToStore(suffix, sink.s, PathHash(), references);
-    
-    if (!readOnlyMode) addTempRoot(dstPath);
-
-    if (!readOnlyMode && !isValidPath(dstPath)) {
-
-        PathSet lockPaths;
-        lockPaths.insert(dstPath);
-        PathLocks outputLock(lockPaths);
-
-        if (!isValidPath(dstPath)) {
-
-            if (pathExists(dstPath)) deletePath(dstPath);
-
-            writeStringToFile(dstPath, s);
-
-            canonicalisePathMetaData(dstPath);
-            
-            Transaction txn(nixDB);
-            registerValidPath(txn, dstPath,
-                hashPath(htSHA256, dstPath), references, "");
-            txn.commit();
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
 }
 
 
@@ -751,6 +900,7 @@ void deleteFromStore(const Path & _path)
 
 void verifyStore(bool checkContents)
 {
+#if 0    
     Transaction txn(nixDB);
 
     Paths paths;
@@ -891,6 +1041,7 @@ void verifyStore(bool checkContents)
     }
 
     txn.commit();
+#endif    
 }
 
 
@@ -902,6 +1053,7 @@ void verifyStore(bool checkContents)
 static void upgradeStore()
 {
     printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+#if 0    
 
     Transaction txn(nixDB);
 
@@ -982,4 +1134,5 @@ static void upgradeStore()
 
     /* !!! maybe this transaction is way too big */
     txn.commit();
+#endif    
 }

src/libstore/store.hh

@@ -12,6 +12,27 @@ using namespace std;
 const int nixSchemaVersion = 2;
 
 
+/* Path hashes are the hash components of store paths, e.g., the
+   `zvhgns772jpj68l40mq1jb74wpfsf0ma' in
+   `/nix/store/zvhgns772jpj68l40mq1jb74wpfsf0ma-glibc'.  These are
+   truncated SHA-256 hashes of the path contents,  */
+struct PathHash
+{
+private:
+    string rep;
+public:
+    PathHash();
+    PathHash(const Hash & h);
+    PathHash(const string & h);
+    string toString() const;
+    bool PathHash::isNull() const;
+    bool operator ==(const PathHash & hash2) const;
+    bool operator <(const PathHash & hash2) const;
+};
+
+
+
+#if 0
 /* A substitute is a program invocation that constructs some store
    path (typically by fetching it from somewhere, e.g., from the
    network). */
@@ -32,6 +53,37 @@ struct Substitute
 };
 
 typedef list<Substitute> Substitutes;
+#endif
+
+
+/* A trust identifier, which is a name of an entity involved in a
+   trust relation.  Right now this is just a user ID (e.g.,
+   `root'). */
+typedef string TrustId;
+
+
+/* An output path equivalence class.  They represent outputs of
+   derivations.  That is, a derivation can have several outputs (e.g.,
+   `out', `lib', `man', etc.), each of which maps to a output path
+   equivalence class.  They can map to a number of concrete paths,
+   depending on what users built the derivation.
+
+   Equivalence classes are actually "placeholder" store paths that
+   never get built.  They do occur in derivations however in
+   command-line arguments and environment variables, but get
+   substituted with concrete paths when we actually build. */
+typedef Path OutputEqClass;
+
+
+/* A member of an output path equivalence class, i.e., a store path
+   that has been produced by a certain derivation. */
+struct OutputEqMember
+{
+    TrustId trustId;
+    Path path;
+};
+
+typedef list<OutputEqMember> OutputEqMembers;
 
 
 /* Open the database environment. */
@@ -43,9 +95,12 @@ void initDB();
 /* Get a transaction object. */
 void createStoreTransaction(Transaction & txn);
 
+
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
+
+#if 0
 /* Register a substitute. */
 void registerSubstitute(const Transaction & txn,
     const Path & srcPath, const Substitute & sub);
@@ -55,6 +110,8 @@ Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
 
 /* Deregister all substitutes. */
 void clearSubstitutes();
+#endif
+
 
 /* Register the validity of a path, i.e., that `path' exists, that the
    paths referenced by it exists, and in the case of an output path of
@@ -79,6 +136,7 @@ typedef list<ValidPathInfo> ValidPathInfos;
 void registerValidPaths(const Transaction & txn,
     const ValidPathInfos & infos);
 
+
 /* Throw an exception if `path' is not directly in the Nix store. */
 void assertStorePath(const Path & path);
 
@@ -91,6 +149,11 @@ void checkStoreName(const string & name);
    /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
 Path toStorePath(const Path & path);
 
+PathHash hashPartOf(const Path & path);
+
+string namePartOf(const Path & path);
+
+
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 0 (i.e.,
@@ -105,6 +168,7 @@ void canonicalisePathMetaData(const Path & path);
 bool isValidPathTxn(const Transaction & txn, const Path & path);
 bool isValidPath(const Path & path);
 
+
 /* Queries the hash of a valid path. */ 
 Hash queryPathHash(const Path & path);
 
@@ -123,6 +187,10 @@ void queryReferences(const Transaction & txn,
 void queryReferers(const Transaction & txn,
     const Path & storePath, PathSet & referers);
 
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members);
+    
+#if 0
 /* Sets the deriver of a store path.  Use with care! */
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver);
@@ -130,27 +198,47 @@ void setDeriver(const Transaction & txn, const Path & storePath,
 /* Query the deriver of a store path.  Return the empty string if no
    deriver has been set. */
 Path queryDeriver(const Transaction & txn, const Path & storePath);
+#endif
+
 
 /* Constructs a unique store path name. */
-Path makeStorePath(const string & type,
+void makeStorePath(const Hash & contentHash, const string & suffix,
-    const Hash & hash, const string & suffix);
+    Path & path, PathHash & pathHash);
+
+/* Constructs a random store path name.  Only to be used for temporary
+   build outputs, since these will violate the hash invariant. */
+Path makeRandomStorePath(const string & suffix);
+
+
+/* Hash rewriting. */
+typedef map<PathHash, PathHash> HashRewrites;
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions);
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites);
+
 
 /* Copy the contents of a path to the store and register the validity
    the resulting path.  The resulting path is returned. */
-Path addToStore(const Path & srcPath);
+Path addToStore(const Path & srcPath, const PathHash & selfHash = PathHash(),
+    const string & suffix = "");
 
+#if 0
 /* Like addToStore(), but for pre-adding the outputs of fixed-output
    derivations. */
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
 
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name);
+#endif
 
 /* Like addToStore, but the contents written to the output path is a
    regular file containing the given string. */
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references);
 
+
 /* Delete a value from the nixStore directory. */
 void deleteFromStore(const Path & path);
 

src/libutil/archive.cc

@@ -140,6 +140,26 @@ void dumpPath(const Path & path, DumpSink & sink)
 }
 
 
+void makeSingletonArchive(const string & contents, DumpSink & sink)
+{
+    /* !!! hacky; have to keep this synchronised with dumpPath().  It
+       would be better to parameterise dumpPath() with a file system
+       "traverser". */
+    writeString(archiveVersion1, sink);
+    writeString("(", sink);
+    writeString("type", sink);
+    writeString("regular", sink);
+
+    unsigned int size = contents.size();
+    writeString("contents", sink);
+    writeInt(size, sink);
+    sink((const unsigned char *) contents.c_str(), size);
+    writePadding(size, sink);
+
+    writeString(")", sink);
+}
+
+
 static Error badArchive(string s)
 {
     return Error("bad archive: " + s);

src/libutil/archive.hh

@@ -48,6 +48,10 @@ struct DumpSink
 
 void dumpPath(const Path & path, DumpSink & sink);
 
+/* Make an archive consisting of a single non-executable regular
+   file, with specified string contents. */
+void makeSingletonArchive(const string & contents, DumpSink & sink);
+
 
 struct RestoreSource
 {

src/nix-env/main.cc

@@ -735,12 +735,14 @@ static void opQuery(Globals & globals,
         Strings columns;
         
         if (printStatus) {
+#if 0            
             Substitutes subs = querySubstitutes(noTxn, i->queryDrvPath(globals.state));
+#endif            
             columns.push_back(
                 (string) (installed.find(i->queryOutPath(globals.state))
                     != installed.end() ? "I" : "-")
                 + (isValidPath(i->queryOutPath(globals.state)) ? "P" : "-")
-                + (subs.size() > 0 ? "S" : "-"));
+                + (/* XXX subs.size() > 0 */ false ? "S" : "-"));
         }
 
         if (printName) columns.push_back(i->name);

src/nix-store/main.cc

@@ -96,6 +96,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
 }
 
 
+#if 0
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
@@ -141,8 +142,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
     cout << format("%1%\n") %
         makeFixedOutputPath(recursive, hashAlgo, h, name);
 }
+#endif
 
 
+#if 0
 /* Place in `paths' the set of paths that are required to `realise'
    the given store path, i.e., all paths necessary for valid
    deployment of the path.  For a derivation, this is the union of
@@ -393,8 +396,10 @@ static void opQuery(Strings opFlags, Strings opArgs)
             abort();
     }
 }
+#endif
 
 
+#if 0
 static void opRegisterSubstitutes(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -441,6 +446,7 @@ static void opClearSubstitutes(Strings opFlags, Strings opArgs)
 
     clearSubstitutes();
 }
+#endif
 
 
 static void opRegisterValidity(Strings opFlags, Strings opArgs)
@@ -602,6 +608,7 @@ void run(Strings args)
             op = opRealise;
         else if (arg == "--add" || arg == "-A")
             op = opAdd;
+#if 0        
         else if (arg == "--add-fixed")
             op = opAddFixed;
         else if (arg == "--print-fixed-path")
@@ -612,6 +619,7 @@ void run(Strings args)
             op = opRegisterSubstitutes;
         else if (arg == "--clear-substitutes")
             op = opClearSubstitutes;
+#endif            
         else if (arg == "--register-validity")
             op = opRegisterValidity;
         else if (arg == "--check-validity")