Merge remote-tracking branch 'upstream/master' into better-ca-parse-errors

src/libexpr/primops.cc

@@ -1199,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     string name;
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
-    Hash expectedHash(htSHA256);
+    std::optional<Hash> expectedHash;
 
     for (auto & attr : *args[0]->attrs) {
         const string & n(attr.name);

src/libstore/binary-cache-store.cc

@@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 
     auto [fdTemp, fnTemp] = createTempFile();
 
+    AutoDelete autoDelete(fnTemp);
+
     auto now1 = std::chrono::steady_clock::now();
 
     /* Read the NAR simultaneously into a CompressionSink+FileSink (to
@@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     TeeSource teeSource(narSource, *compressionSink);
     narAccessor = makeNarAccessor(teeSource);
     compressionSink->finish();
+    fileSink.flush();
     }
 
     auto now2 = std::chrono::steady_clock::now();
@@ -280,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
         upsertFile(narInfo->url,
-            std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
+            std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
             "application/x-nix-nar");
     } else
         stats.narWriteAverted++;

src/libstore/local-store.cc

@@ -1038,20 +1038,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
-    Path srcPath(absPath(_srcPath));
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
-        else
-            readFile(srcPath, sink);
-    });
-    return addToStoreFromDump(*source, name, method, hashAlgo, repair);
-}
-
-
 StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
     FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
 {

src/libstore/local-store.hh

@@ -145,16 +145,8 @@ public:
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStore(const string & name, const Path & srcPath,
-        FileIngestionMethod method, HashType hashAlgo,
-        PathFilter & filter, RepairFlag repair) override;
-
-    /* Like addToStore(), but the contents of the path are contained
-       in `dump', which is either a NAR serialisation (if recursive ==
-       true) or simply the contents of a regular file (if recursive ==
-       false). */
     StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;

src/libstore/path-info.hh

@@ -0,0 +1,111 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+
+#include <string>
+#include <optional>
+
+namespace nix {
+
+
+class Store;
+
+
+struct SubstitutablePathInfo
+{
+    std::optional<StorePath> deriver;
+    StorePathSet references;
+    uint64_t downloadSize; /* 0 = unknown or inapplicable */
+    uint64_t narSize; /* 0 = unknown */
+};
+
+typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
+
+
+struct ValidPathInfo
+{
+    StorePath path;
+    std::optional<StorePath> deriver;
+    // TODO document this
+    std::optional<Hash> narHash;
+    StorePathSet references;
+    time_t registrationTime = 0;
+    uint64_t narSize = 0; // 0 = unknown
+    uint64_t id; // internal use only
+
+    /* Whether the path is ultimately trusted, that is, it's a
+       derivation output that was built locally. */
+    bool ultimate = false;
+
+    StringSet sigs; // note: not necessarily verified
+
+    /* If non-empty, an assertion that the path is content-addressed,
+       i.e., that the store path is computed from a cryptographic hash
+       of the contents of the path, plus some other bits of data like
+       the "name" part of the path. Such a path doesn't need
+       signatures, since we don't have to trust anybody's claim that
+       the path is the output of a particular derivation. (In the
+       extensional store model, we have to trust that the *contents*
+       of an output path of a derivation were actually produced by
+       that derivation. In the intensional model, we have to trust
+       that a particular output path was produced by a derivation; the
+       path then implies the contents.)
+
+       Ideally, the content-addressability assertion would just be a Boolean,
+       and the store path would be computed from the name component, ‘narHash’
+       and ‘references’. However, we support many types of content addresses.
+    */
+    std::optional<ContentAddress> ca;
+
+    bool operator == (const ValidPathInfo & i) const
+    {
+        return
+            path == i.path
+            && narHash == i.narHash
+            && references == i.references;
+    }
+
+    /* Return a fingerprint of the store path to be used in binary
+       cache signatures. It contains the store path, the base-32
+       SHA-256 hash of the NAR serialisation of the path, the size of
+       the NAR, and the sorted references. The size field is strictly
+       speaking superfluous, but might prevent endless/excessive data
+       attacks. */
+    std::string fingerprint(const Store & store) const;
+
+    void sign(const Store & store, const SecretKey & secretKey);
+
+    /* Return true iff the path is verifiably content-addressed. */
+    bool isContentAddressed(const Store & store) const;
+
+    /* Functions to view references + hasSelfReference as one set, mainly for
+       compatibility's sake. */
+    StorePathSet referencesPossiblyToSelf() const;
+    void insertReferencePossiblyToSelf(StorePath && ref);
+    void setReferencesPossiblyToSelf(StorePathSet && refs);
+
+    static const size_t maxSigs = std::numeric_limits<size_t>::max();
+
+    /* Return the number of signatures on this .narinfo that were
+       produced by one of the specified keys, or maxSigs if the path
+       is content-addressed. */
+    size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
+
+    /* Verify a single signature. */
+    bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
+
+    Strings shortRefs() const;
+
+    ValidPathInfo(const ValidPathInfo & other) = default;
+
+    ValidPathInfo(StorePath && path) : path(std::move(path)) { };
+    ValidPathInfo(const StorePath & path) : path(path) { };
+
+    virtual ~ValidPathInfo() { }
+};
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+}

src/libstore/s3-binary-cache-store.cc

@@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const std::string & mimeType,
         const std::string & contentEncoding)
     {
+        istream->seekg(0, istream->end);
+        auto size = istream->tellg();
+        istream->seekg(0, istream->beg);
+
         auto maxThreads = std::thread::hardware_concurrency();
 
         static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -343,10 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
                 .count();
 
-        printInfo("uploaded 's3://%s/%s' in %d ms",
-            bucketName, path, duration);
+        printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
+            bucketName, path, size, duration);
 
         stats.putTimeMs += duration;
+        stats.putBytes += std::max(size, (decltype(size)) 0);
         stats.put++;
     }
 

src/libstore/s3-binary-cache-store.hh

@@ -19,6 +19,7 @@ public:
     struct Stats
     {
         std::atomic<uint64_t> put{0};
+        std::atomic<uint64_t> putBytes{0};
         std::atomic<uint64_t> putTimeMs{0};
         std::atomic<uint64_t> get{0};
         std::atomic<uint64_t> getBytes{0};

src/libstore/store-api.cc

@@ -239,6 +239,20 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 }
 
 
+StorePath Store::addToStore(const string & name, const Path & _srcPath,
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+{
+    Path srcPath(absPath(_srcPath));
+    auto source = sinkToSource([&](Sink & sink) {
+        if (method == FileIngestionMethod::Recursive)
+            dumpPath(srcPath, sink, filter);
+        else
+            readFile(srcPath, sink);
+    });
+    return addToStoreFromDump(*source, name, method, hashAlgo, repair);
+}
+
+
 /*
 The aim of this function is to compute in one pass the correct ValidPathInfo for
 the files that we are trying to add to the store. To accomplish that in one

src/libstore/store-api.hh

@@ -10,6 +10,7 @@
 #include "globals.hh"
 #include "config.hh"
 #include "derivations.hh"
+#include "path-info.hh"
 
 #include <atomic>
 #include <limits>
@@ -101,95 +102,6 @@ struct GCResults
 };
 
 
-struct SubstitutablePathInfo
-{
-    std::optional<StorePath> deriver;
-    StorePathSet references;
-    uint64_t downloadSize; /* 0 = unknown or inapplicable */
-    uint64_t narSize; /* 0 = unknown */
-};
-
-typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
-
-struct ValidPathInfo
-{
-    StorePath path;
-    std::optional<StorePath> deriver;
-    // TODO document this
-    std::optional<Hash> narHash;
-    StorePathSet references;
-    time_t registrationTime = 0;
-    uint64_t narSize = 0; // 0 = unknown
-    uint64_t id; // internal use only
-
-    /* Whether the path is ultimately trusted, that is, it's a
-       derivation output that was built locally. */
-    bool ultimate = false;
-
-    StringSet sigs; // note: not necessarily verified
-
-    /* If non-empty, an assertion that the path is content-addressed,
-       i.e., that the store path is computed from a cryptographic hash
-       of the contents of the path, plus some other bits of data like
-       the "name" part of the path. Such a path doesn't need
-       signatures, since we don't have to trust anybody's claim that
-       the path is the output of a particular derivation. (In the
-       extensional store model, we have to trust that the *contents*
-       of an output path of a derivation were actually produced by
-       that derivation. In the intensional model, we have to trust
-       that a particular output path was produced by a derivation; the
-       path then implies the contents.)
-
-       Ideally, the content-addressability assertion would just be a Boolean,
-       and the store path would be computed from the name component, ‘narHash’
-       and ‘references’. However, we support many types of content addresses.
-    */
-    std::optional<ContentAddress> ca;
-
-    bool operator == (const ValidPathInfo & i) const
-    {
-        return
-            path == i.path
-            && narHash == i.narHash
-            && references == i.references;
-    }
-
-    /* Return a fingerprint of the store path to be used in binary
-       cache signatures. It contains the store path, the base-32
-       SHA-256 hash of the NAR serialisation of the path, the size of
-       the NAR, and the sorted references. The size field is strictly
-       speaking superfluous, but might prevent endless/excessive data
-       attacks. */
-    std::string fingerprint(const Store & store) const;
-
-    void sign(const Store & store, const SecretKey & secretKey);
-
-    /* Return true iff the path is verifiably content-addressed. */
-    bool isContentAddressed(const Store & store) const;
-
-    static const size_t maxSigs = std::numeric_limits<size_t>::max();
-
-    /* Return the number of signatures on this .narinfo that were
-       produced by one of the specified keys, or maxSigs if the path
-       is content-addressed. */
-    size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
-
-    /* Verify a single signature. */
-    bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
-
-    Strings shortRefs() const;
-
-    ValidPathInfo(const ValidPathInfo & other) = default;
-
-    ValidPathInfo(StorePath && path) : path(std::move(path)) { };
-    ValidPathInfo(const StorePath & path) : path(path) { };
-
-    virtual ~ValidPathInfo() { }
-};
-
-typedef list<ValidPathInfo> ValidPathInfos;
-
-
 enum BuildMode { bmNormal, bmRepair, bmCheck };
 
 
@@ -456,7 +368,7 @@ public:
        libutil/archive.hh). */
     virtual StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path, using a constant amount of
@@ -465,6 +377,10 @@ public:
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         std::optional<Hash> expectedCAHash = {});
 
+    /* Like addToStore(), but the contents of the path are contained
+       in `dump', which is either a NAR serialisation (if recursive ==
+       true) or simply the contents of a regular file (if recursive ==
+       false). */
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(Source & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

src/libutil/util.cc

@@ -494,6 +494,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 {
     Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
     // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
     if (!fd)
         throw SysError("creating temporary file '%s'", tmpl);

src/nix/build.cc

@@ -9,6 +9,7 @@ using namespace nix;
 struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 {
     Path outLink = "result";
+    BuildMode buildMode = bmNormal;
 
     CmdBuild()
     {
@@ -26,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
             .description = "do not create a symlink to the build result",
             .handler = {&outLink, Path("")},
         });
+
+        addFlag({
+            .longName = "rebuild",
+            .description = "rebuild an already built package and compare the result to the existing store paths",
+            .handler = {&buildMode, bmCheck},
+        });
     }
 
     std::string description() override
@@ -53,7 +60,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 
     void run(ref<Store> store) override
     {
-        auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables);
+        auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode);
 
         if (dryRun) return;
 

src/nix/bundle.cc

@@ -0,0 +1,129 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "fs-accessor.hh"
+
+using namespace nix;
+
+struct CmdBundle : InstallableCommand
+{
+    std::string bundler = "github:matthewbauer/nix-bundle";
+    std::optional<Path> outLink;
+
+    CmdBundle()
+    {
+        addFlag({
+            .longName = "bundler",
+            .description = "use custom bundler",
+            .labels = {"flake-url"},
+            .handler = {&bundler},
+            .completer = {[&](size_t, std::string_view prefix) {
+                completeFlakeRef(getStore(), prefix);
+            }}
+        });
+
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "path of the symlink to the build result",
+            .labels = {"path"},
+            .handler = {&outLink},
+            .completer = completePath
+        });
+    }
+
+    std::string description() override
+    {
+        return "bundle an application so that it works outside of the Nix store";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To bundle Hello:",
+                "nix bundle hello"
+            },
+        };
+    }
+
+    Category category() override { return catSecondary; }
+
+    Strings getDefaultFlakeAttrPaths() override
+    {
+        Strings res{"defaultApp." + settings.thisSystem.get()};
+        for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths())
+            res.push_back(s);
+        return res;
+    }
+
+    Strings getDefaultFlakeAttrPathPrefixes() override
+    {
+        Strings res{"apps." + settings.thisSystem.get() + ".", "packages"};
+        for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes())
+            res.push_back(s);
+        return res;
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto evalState = getEvalState();
+
+        auto app = installable->toApp(*evalState);
+        store->buildPaths(app.context);
+
+        auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+        const flake::LockFlags lockFlags{ .writeLockFile = false };
+        auto bundler = InstallableFlake(
+            evalState, std::move(bundlerFlakeRef),
+            Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
+            Strings({"bundlers."}), lockFlags);
+
+        Value * arg = evalState->allocValue();
+        evalState->mkAttrs(*arg, 2);
+
+        PathSet context;
+        for (auto & i : app.context)
+            context.insert("=" + store->printStorePath(i.path));
+        mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context);
+
+        mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
+
+        arg->attrs->sort();
+ 
+        auto vRes = evalState->allocValue();
+        evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
+
+        if (!evalState->isDerivation(*vRes))
+            throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+        auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+        if (!attr1)
+            throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+        PathSet context2;
+        StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
+
+        auto attr2 = vRes->attrs->find(evalState->sOutPath);
+        if (!attr2)
+            throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+        StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
+
+        store->buildPaths({{drvPath}});
+
+        auto outPathS = store->printStorePath(outPath);
+
+        auto info = store->queryPathInfo(outPath);
+        if (!info->references.empty())
+            throw Error("'%s' has references; a bundler must not leave any references", outPathS);
+
+        if (!outLink)
+            outLink = baseNameOf(app.program);
+
+        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true);
+    }
+};
+
+static auto r2 = registerCommand<CmdBundle>("bundle");

src/nix/command.hh

@@ -5,6 +5,7 @@
 #include "common-eval-args.hh"
 #include "path.hh"
 #include "flake/lockfile.hh"
+#include "store-api.hh"
 
 #include <optional>
 
@@ -185,7 +186,7 @@ static RegisterCommand registerCommand(const std::string & name)
 }
 
 Buildables build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables);
+    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
 
 std::set<StorePath> toStorePaths(ref<Store> store,
     Realise mode, OperateOn operateOn,

src/nix/flake.cc

@@ -368,6 +368,21 @@ struct CmdFlakeCheck : FlakeCommand
             }
         };
 
+        auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+            try {
+                state->forceValue(v, pos);
+                if (v.type != tLambda)
+                    throw Error("bundler must be a function");
+                if (!v.lambda.fun->formals ||
+                    v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
+                    v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end())
+                    throw Error("bundler must take formal arguments 'program' and 'system'");
+            } catch (Error & e) {
+                e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath));
+                throw;
+            }
+        };
+
         {
             Activity act(*logger, lvlInfo, actUnknown, "evaluating flake");
 
@@ -490,6 +505,16 @@ struct CmdFlakeCheck : FlakeCommand
                                     *attr.value, *attr.pos);
                         }
 
+                        else if (name == "defaultBundler")
+                            checkBundler(name, vOutput, pos);
+
+                        else if (name == "bundlers") {
+                            state->forceAttrs(vOutput, pos);
+                            for (auto & attr : *vOutput.attrs)
+                                checkBundler(fmt("%s.%s", name, attr.name),
+                                    *attr.value, *attr.pos);
+                        }
+
                         else
                             warn("unknown flake output '%s'", name);
 

src/nix/installables.cc

@@ -645,7 +645,7 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
 }
 
 Buildables build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables)
+    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
@@ -671,7 +671,7 @@ Buildables build(ref<Store> store, Realise mode,
     if (mode == Realise::Nothing)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild);
+        store->buildPaths(pathsToBuild, bMode);
 
     return buildables;
 }