mirror of
https://github.com/NixOS/nix.git
synced 2025-11-20 17:29:36 +01:00
Merge branch 'master' into read-only-local-store
This commit is contained in:
Ben Radford
commit
abb3bb7133
52 changed files with 851 additions and 272 deletions
|
|
@ -4,7 +4,7 @@
|
|||
#include "worker.hh"
|
||||
#include "builtins.hh"
|
||||
#include "builtins/buildenv.hh"
|
||||
#include "references.hh"
|
||||
#include "path-references.hh"
|
||||
#include "finally.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -1457,7 +1457,7 @@ void LocalDerivationGoal::startDaemon()
|
|||
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
|
||||
if (!remote) {
|
||||
if (errno == EINTR || errno == EAGAIN) continue;
|
||||
if (errno == EINVAL) break;
|
||||
if (errno == EINVAL || errno == ECONNABORTED) break;
|
||||
throw SysError("accepting connection");
|
||||
}
|
||||
|
||||
|
|
@ -1487,8 +1487,22 @@ void LocalDerivationGoal::startDaemon()
|
|||
|
||||
void LocalDerivationGoal::stopDaemon()
|
||||
{
|
||||
if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
|
||||
throw SysError("shutting down daemon socket");
|
||||
if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) {
|
||||
// According to the POSIX standard, the 'shutdown' function should
|
||||
// return an ENOTCONN error when attempting to shut down a socket that
|
||||
// hasn't been connected yet. This situation occurs when the 'accept'
|
||||
// function is called on a socket without any accepted connections,
|
||||
// leaving the socket unconnected. While Linux doesn't seem to produce
|
||||
// an error for sockets that have only been accepted, more
|
||||
// POSIX-compliant operating systems like OpenBSD, macOS, and others do
|
||||
// return the ENOTCONN error. Therefore, we handle this error here to
|
||||
// avoid raising an exception for compliant behaviour.
|
||||
if (errno == ENOTCONN) {
|
||||
daemonSocket.close();
|
||||
} else {
|
||||
throw SysError("shutting down daemon socket");
|
||||
}
|
||||
}
|
||||
|
||||
if (daemonThread.joinable())
|
||||
daemonThread.join();
|
||||
|
|
@ -1499,7 +1513,8 @@ void LocalDerivationGoal::stopDaemon()
|
|||
thread.join();
|
||||
daemonWorkerThreads.clear();
|
||||
|
||||
daemonSocket = -1;
|
||||
// release the socket.
|
||||
daemonSocket.close();
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -2379,18 +2394,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
continue;
|
||||
auto references = *referencesOpt;
|
||||
|
||||
auto rewriteOutput = [&]() {
|
||||
auto rewriteOutput = [&](const StringMap & rewrites) {
|
||||
/* Apply hash rewriting if necessary. */
|
||||
if (!outputRewrites.empty()) {
|
||||
if (!rewrites.empty()) {
|
||||
debug("rewriting hashes in '%1%'; cross fingers", actualPath);
|
||||
|
||||
/* FIXME: this is in-memory. */
|
||||
StringSink sink;
|
||||
dumpPath(actualPath, sink);
|
||||
/* FIXME: Is this actually streaming? */
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink(rewrites, nextSink);
|
||||
dumpPath(actualPath, rsink);
|
||||
rsink.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
restorePath(tmpPath, *source);
|
||||
deletePath(actualPath);
|
||||
sink.s = rewriteStrings(sink.s, outputRewrites);
|
||||
StringSource source(sink.s);
|
||||
restorePath(actualPath, source);
|
||||
movePath(tmpPath, actualPath);
|
||||
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
we don't have to do another traversal. */
|
||||
|
|
@ -2439,7 +2457,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
"since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
|
||||
actualPath);
|
||||
}
|
||||
rewriteOutput();
|
||||
rewriteOutput(outputRewrites);
|
||||
/* FIXME optimize and deduplicate with addToStore */
|
||||
std::string oldHashPart { scratchPath->hashPart() };
|
||||
HashModuloSink caSink { outputHash.hashType, oldHashPart };
|
||||
|
|
@ -2477,16 +2495,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
Hash::dummy,
|
||||
};
|
||||
if (*scratchPath != newInfo0.path) {
|
||||
// Also rewrite the output path
|
||||
auto source = sinkToSource([&](Sink & nextSink) {
|
||||
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
|
||||
dumpPath(actualPath, rsink2);
|
||||
rsink2.flush();
|
||||
});
|
||||
Path tmpPath = actualPath + ".tmp";
|
||||
restorePath(tmpPath, *source);
|
||||
deletePath(actualPath);
|
||||
movePath(tmpPath, actualPath);
|
||||
// If the path has some self-references, we need to rewrite
|
||||
// them.
|
||||
// (note that this doesn't invalidate the ca hash we calculated
|
||||
// above because it's computed *modulo the self-references*, so
|
||||
// it already takes this rewrite into account).
|
||||
rewriteOutput(
|
||||
StringMap{{oldHashPart,
|
||||
std::string(newInfo0.path.hashPart())}});
|
||||
}
|
||||
|
||||
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
|
||||
|
|
@ -2508,7 +2524,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
outputRewrites.insert_or_assign(
|
||||
std::string { scratchPath->hashPart() },
|
||||
std::string { requiredFinalPath.hashPart() });
|
||||
rewriteOutput();
|
||||
rewriteOutput(outputRewrites);
|
||||
auto narHashAndSize = hashPath(htSHA256, actualPath);
|
||||
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
|
||||
newInfo0.narSize = narHashAndSize.second;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ void setPersonality(std::string_view system)
|
|||
&& (std::string_view(SYSTEM) == "x86_64-linux"
|
||||
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|
||||
|| system == "armv7l-linux"
|
||||
|| system == "armv6l-linux")
|
||||
|| system == "armv6l-linux"
|
||||
|| system == "armv5tel-linux")
|
||||
{
|
||||
if (personality(PER_LINUX32) == -1)
|
||||
throw SysError("cannot set 32-bit personality");
|
||||
|
|
|
|||
|
|
@ -691,20 +691,19 @@ public:
|
|||
Strings{"https://cache.nixos.org/"},
|
||||
"substituters",
|
||||
R"(
|
||||
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
|
||||
to be used as substituters, separated by whitespace.
|
||||
Substituters are tried based on their Priority value, which each substituter can set
|
||||
independently. Lower value means higher priority.
|
||||
The default is `https://cache.nixos.org`, with a Priority of 40.
|
||||
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
|
||||
A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
|
||||
|
||||
At least one of the following conditions must be met for Nix to use
|
||||
a substituter:
|
||||
Substituters are tried based on their priority value, which each substituter can set independently.
|
||||
Lower value means higher priority.
|
||||
The default is `https://cache.nixos.org`, which has a priority of 40.
|
||||
|
||||
At least one of the following conditions must be met for Nix to use a substituter:
|
||||
|
||||
- the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
|
||||
- the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list
|
||||
|
||||
In addition, each store path should be trusted as described
|
||||
in [`trusted-public-keys`](#conf-trusted-public-keys)
|
||||
In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys)
|
||||
)",
|
||||
{"binary-caches"}};
|
||||
|
||||
|
|
@ -896,12 +895,11 @@ public:
|
|||
this, {}, "hashed-mirrors",
|
||||
R"(
|
||||
A list of web servers used by `builtins.fetchurl` to obtain files by
|
||||
hash. The default is `http://tarballs.nixos.org/`. Given a hash type
|
||||
*ht* and a base-16 hash *h*, Nix will try to download the file from
|
||||
*hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
|
||||
they have disappeared from their original URI. For example, given
|
||||
the default mirror `http://tarballs.nixos.org/`, when building the
|
||||
derivation
|
||||
hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
|
||||
download the file from *hashed-mirror*/*ht*/*h*. This allows files to
|
||||
be downloaded even if they have disappeared from their original URI.
|
||||
For example, given an example mirror `http://tarballs.nixos.org/`,
|
||||
when building the derivation
|
||||
|
||||
```nix
|
||||
builtins.fetchurl {
|
||||
|
|
|
|||
73
src/libstore/path-references.cc
Normal file
73
src/libstore/path-references.cc
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
#include "path-references.hh"
|
||||
#include "hash.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <algorithm>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
|
||||
: RefScanSink(std::move(hashes))
|
||||
, backMap(std::move(backMap))
|
||||
{ }
|
||||
|
||||
PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
|
||||
{
|
||||
StringSet hashes;
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
for (auto & i : refs) {
|
||||
std::string hashPart(i.hashPart());
|
||||
auto inserted = backMap.emplace(hashPart, i).second;
|
||||
assert(inserted);
|
||||
hashes.insert(hashPart);
|
||||
}
|
||||
|
||||
return PathRefScanSink(std::move(hashes), std::move(backMap));
|
||||
}
|
||||
|
||||
StorePathSet PathRefScanSink::getResultPaths()
|
||||
{
|
||||
/* Map the hashes found back to their store paths. */
|
||||
StorePathSet found;
|
||||
for (auto & i : getResult()) {
|
||||
auto j = backMap.find(i);
|
||||
assert(j != backMap.end());
|
||||
found.insert(j->second);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(
|
||||
const std::string & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
HashSink hashSink { htSHA256 };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<StorePathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
StorePathSet scanForReferences(
|
||||
Sink & toTee,
|
||||
const Path & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
|
||||
TeeSink sink { refsSink, toTee };
|
||||
|
||||
/* Look for the hashes in the NAR dump of the path. */
|
||||
dumpPath(path, sink);
|
||||
|
||||
return refsSink.getResultPaths();
|
||||
}
|
||||
|
||||
}
|
||||
25
src/libstore/path-references.hh
Normal file
25
src/libstore/path-references.hh
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#pragma once
|
||||
|
||||
#include "references.hh"
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
|
||||
|
||||
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
|
||||
|
||||
class PathRefScanSink : public RefScanSink
|
||||
{
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
|
||||
|
||||
public:
|
||||
|
||||
static PathRefScanSink fromPaths(const StorePathSet & refs);
|
||||
|
||||
StorePathSet getResultPaths();
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,188 +0,0 @@
|
|||
#include "references.hh"
|
||||
#include "hash.hh"
|
||||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
static size_t refLength = 32; /* characters */
|
||||
|
||||
|
||||
static void search(
|
||||
std::string_view s,
|
||||
StringSet & hashes,
|
||||
StringSet & seen)
|
||||
{
|
||||
static std::once_flag initialised;
|
||||
static bool isBase32[256];
|
||||
std::call_once(initialised, [](){
|
||||
for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
|
||||
for (unsigned int i = 0; i < base32Chars.size(); ++i)
|
||||
isBase32[(unsigned char) base32Chars[i]] = true;
|
||||
});
|
||||
|
||||
for (size_t i = 0; i + refLength <= s.size(); ) {
|
||||
int j;
|
||||
bool match = true;
|
||||
for (j = refLength - 1; j >= 0; --j)
|
||||
if (!isBase32[(unsigned char) s[i + j]]) {
|
||||
i += j + 1;
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
if (!match) continue;
|
||||
std::string ref(s.substr(i, refLength));
|
||||
if (hashes.erase(ref)) {
|
||||
debug("found reference to '%1%' at offset '%2%'", ref, i);
|
||||
seen.insert(ref);
|
||||
}
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void RefScanSink::operator () (std::string_view data)
|
||||
{
|
||||
/* It's possible that a reference spans the previous and current
|
||||
fragment, so search in the concatenation of the tail of the
|
||||
previous fragment and the start of the current fragment. */
|
||||
auto s = tail;
|
||||
auto tailLen = std::min(data.size(), refLength);
|
||||
s.append(data.data(), tailLen);
|
||||
search(s, hashes, seen);
|
||||
|
||||
search(data, hashes, seen);
|
||||
|
||||
auto rest = refLength - tailLen;
|
||||
if (rest < tail.size())
|
||||
tail = tail.substr(tail.size() - rest);
|
||||
tail.append(data.data() + data.size() - tailLen, tailLen);
|
||||
}
|
||||
|
||||
|
||||
PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
|
||||
: RefScanSink(std::move(hashes))
|
||||
, backMap(std::move(backMap))
|
||||
{ }
|
||||
|
||||
PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
|
||||
{
|
||||
StringSet hashes;
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
for (auto & i : refs) {
|
||||
std::string hashPart(i.hashPart());
|
||||
auto inserted = backMap.emplace(hashPart, i).second;
|
||||
assert(inserted);
|
||||
hashes.insert(hashPart);
|
||||
}
|
||||
|
||||
return PathRefScanSink(std::move(hashes), std::move(backMap));
|
||||
}
|
||||
|
||||
StorePathSet PathRefScanSink::getResultPaths()
|
||||
{
|
||||
/* Map the hashes found back to their store paths. */
|
||||
StorePathSet found;
|
||||
for (auto & i : getResult()) {
|
||||
auto j = backMap.find(i);
|
||||
assert(j != backMap.end());
|
||||
found.insert(j->second);
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(
|
||||
const std::string & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
HashSink hashSink { htSHA256 };
|
||||
auto found = scanForReferences(hashSink, path, refs);
|
||||
auto hash = hashSink.finish();
|
||||
return std::pair<StorePathSet, HashResult>(found, hash);
|
||||
}
|
||||
|
||||
StorePathSet scanForReferences(
|
||||
Sink & toTee,
|
||||
const Path & path,
|
||||
const StorePathSet & refs)
|
||||
{
|
||||
PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
|
||||
TeeSink sink { refsSink, toTee };
|
||||
|
||||
/* Look for the hashes in the NAR dump of the path. */
|
||||
dumpPath(path, sink);
|
||||
|
||||
return refsSink.getResultPaths();
|
||||
}
|
||||
|
||||
|
||||
RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
|
||||
: from(from), to(to), nextSink(nextSink)
|
||||
{
|
||||
assert(from.size() == to.size());
|
||||
}
|
||||
|
||||
void RewritingSink::operator () (std::string_view data)
|
||||
{
|
||||
std::string s(prev);
|
||||
s.append(data);
|
||||
|
||||
size_t j = 0;
|
||||
while ((j = s.find(from, j)) != std::string::npos) {
|
||||
matches.push_back(pos + j);
|
||||
s.replace(j, from.size(), to);
|
||||
}
|
||||
|
||||
prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1);
|
||||
|
||||
auto consumed = s.size() - prev.size();
|
||||
|
||||
pos += consumed;
|
||||
|
||||
if (consumed) nextSink(s.substr(0, consumed));
|
||||
}
|
||||
|
||||
void RewritingSink::flush()
|
||||
{
|
||||
if (prev.empty()) return;
|
||||
pos += prev.size();
|
||||
nextSink(prev);
|
||||
prev.clear();
|
||||
}
|
||||
|
||||
HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
|
||||
: hashSink(ht)
|
||||
, rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
|
||||
{
|
||||
}
|
||||
|
||||
void HashModuloSink::operator () (std::string_view data)
|
||||
{
|
||||
rewritingSink(data);
|
||||
}
|
||||
|
||||
HashResult HashModuloSink::finish()
|
||||
{
|
||||
rewritingSink.flush();
|
||||
|
||||
/* Hash the positions of the self-references. This ensures that a
|
||||
NAR with self-references and a NAR with some of the
|
||||
self-references already zeroed out do not produce a hash
|
||||
collision. FIXME: proof. */
|
||||
for (auto & pos : rewritingSink.matches)
|
||||
hashSink(fmt("|%d", pos));
|
||||
|
||||
auto h = hashSink.finish();
|
||||
return {h.first, rewritingSink.pos};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "hash.hh"
|
||||
#include "path.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
|
||||
|
||||
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
|
||||
|
||||
class RefScanSink : public Sink
|
||||
{
|
||||
StringSet hashes;
|
||||
StringSet seen;
|
||||
|
||||
std::string tail;
|
||||
|
||||
public:
|
||||
|
||||
RefScanSink(StringSet && hashes) : hashes(hashes)
|
||||
{ }
|
||||
|
||||
StringSet & getResult()
|
||||
{ return seen; }
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
};
|
||||
|
||||
class PathRefScanSink : public RefScanSink
|
||||
{
|
||||
std::map<std::string, StorePath> backMap;
|
||||
|
||||
PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
|
||||
|
||||
public:
|
||||
|
||||
static PathRefScanSink fromPaths(const StorePathSet & refs);
|
||||
|
||||
StorePathSet getResultPaths();
|
||||
};
|
||||
|
||||
struct RewritingSink : Sink
|
||||
{
|
||||
std::string from, to, prev;
|
||||
Sink & nextSink;
|
||||
uint64_t pos = 0;
|
||||
|
||||
std::vector<uint64_t> matches;
|
||||
|
||||
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
|
||||
void flush();
|
||||
};
|
||||
|
||||
struct HashModuloSink : AbstractHashSink
|
||||
{
|
||||
HashSink hashSink;
|
||||
RewritingSink rewritingSink;
|
||||
|
||||
HashModuloSink(HashType ht, const std::string & modulus);
|
||||
|
||||
void operator () (std::string_view data) override;
|
||||
|
||||
HashResult finish() override;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -13,6 +13,14 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::string UDSRemoteStoreConfig::doc()
|
||||
{
|
||||
return
|
||||
#include "uds-remote-store.md"
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
|
|
|
|||
|
|
@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
|
|||
|
||||
const std::string name() override { return "Local Daemon Store"; }
|
||||
|
||||
std::string doc() override
|
||||
{
|
||||
return
|
||||
#include "uds-remote-store.md"
|
||||
;
|
||||
}
|
||||
std::string doc() override;
|
||||
};
|
||||
|
||||
class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue