diff --git a/Makefile b/Makefile index f472ca7e5..c50d2c40f 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) GLOBAL_CXXFLAGS += -O3 else - GLOBAL_CXXFLAGS += -O0 + GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE endif include mk/lib.mk diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 139067f20..883fc27a7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 80078d8a5..0e1f61baa 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,7 +6,6 @@ #include "symbol-table.hh" #include "config.hh" -#include #include #include #include @@ -65,6 +64,11 @@ typedef std::list SearchPath; void initGC(); +struct RegexCache; + +std::shared_ptr makeRegexCache(); + + class EvalState { public: @@ -120,7 +124,7 @@ private: std::unordered_map resolvedPaths; /* Cache used by prim_match(). */ - std::unordered_map regexCache; + std::shared_ptr regexCache; public: diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6363446f6..d5c2ffe66 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -1,6 +1,7 @@ #include "flakeref.hh" #include "store-api.hh" #include "url.hh" +#include "url-parts.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74846944..78431f000 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -1,5 +1,6 @@ #include "lockfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 76e1a26bf..9ca5ac86d 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -115,6 +115,14 @@ public: { return handle_value(mkString, val.c_str()); } +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) + { + // This function ought to be unreachable + assert(false); + return true; + } +#endif bool start_object(std::size_t len) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 89aedfc92..236b8b5a8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3085,17 +3085,25 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); -/* Match a regular expression against a string and return either - ‘null’ or a list containing substring matches. */ +struct RegexCache +{ + std::unordered_map cache; +}; + +std::shared_ptr makeRegexCache() +{ + return std::make_shared(); +} + void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - auto regex = state.regexCache.find(re); - if (regex == state.regexCache.end()) - regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->cache.find(re); + if (regex == state.regexCache->cache.end()) + regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; PathSet context; const std::string str = state.forceString(*args[1], context, pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index cef85cfef..1a064ed5c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -3,8 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "url.hh" - -#include +#include "url-parts.hh" namespace nix { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5ca0f8521..ad7638d73 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1cc0c5e2e..1737658a7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -3,14 +3,15 @@ #include "fetchers.hh" #include "globals.hh" #include "store-api.hh" +#include "url-parts.hh" #include namespace nix::fetchers { -// A github or gitlab url -const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check -std::regex urlRegex(urlRegexS, std::regex::ECMAScript); +// A github or gitlab host +const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { @@ -50,9 +51,9 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } - else if (name == "url") { - if (!std::regex_match(value, urlRegex)) - throw BadURL("URL '%s' contains an invalid instance url", url.url); + else if (name == "host") { + if (!std::regex_match(value, hostRegex)) + throw BadURL("URL '%s' contains an invalid instance host", url.url); host_url = value; } // FIXME: barf on unsupported attributes @@ -67,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); - if (host_url) input.attrs.insert_or_assign("url", *host_url); + if (host_url) input.attrs.insert_or_assign("host", *host_url); return input; } @@ -77,7 +78,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); @@ -210,7 +211,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); @@ -236,7 +237,7 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b981d4d8e..74332ae3d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,4 +1,5 @@ #include "fetchers.hh" +#include "url-parts.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 3e76ffc4d..d80c2ea7a 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 34f844a18..ebc0bd6a4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "json.hh" #include "thread-pool.hh" +#include "callback.hh" #include #include diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6e55f83d5..07c5bceb2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,6 +17,7 @@ #include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" +#include "callback.hh" #include #include @@ -2949,14 +2950,6 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf goal.addDependency(info.path); } - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override - { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); - goal.addDependency(path); - return path; - } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) override { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 0885c3d0e..90a3ad1f5 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,13 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const { +std::string FixedOutputHash::printMethodAlgo() const +{ return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) { +std::string makeFileIngestionPrefix(const FileIngestionMethod m) +{ switch (m) { case FileIngestionMethod::Flat: return ""; @@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) + hash.to_string(Base32, true); } -std::string renderContentAddress(ContentAddress ca) { +std::string renderContentAddress(ContentAddress ca) +{ return std::visit(overloaded { [](TextHash th) { return "text:" + th.hash.to_string(Base32, true); @@ -37,54 +40,97 @@ std::string renderContentAddress(ContentAddress ca) { }, ca); } -ContentAddress parseContentAddress(std::string_view rawCa) { - auto rest = rawCa; +std::string renderContentAddressMethod(ContentAddressMethod cam) +{ + return std::visit(overloaded { + [](TextHashMethod &th) { + return std::string{"text:"} + printHashType(htSHA256); + }, + [](FixedOutputHashMethod &fshm) { + return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); + } + }, cam); +} + +/* + Parses content address strings up to the hash. + */ +static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +{ + std::string_view wholeInput { rest }; std::string_view prefix; { auto optPrefix = splitPrefixTo(rest, ':'); if (!optPrefix) - throw UsageError("not a content address because it is not in the form ':': %s", rawCa); + throw UsageError("not a content address because it is not in the form ':': %s", wholeInput); prefix = *optPrefix; } auto parseHashType_ = [&](){ auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) - throw UsageError("content address hash must be in form ':', but found: %s", rawCa); + throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); return std::move(hashType); }; // Switch on prefix if (prefix == "text") { - // No parsing of the method, "text" only support flat. + // No parsing of the ingestion method, "text" only support flat. HashType hashType = parseHashType_(); if (hashType != htSHA256) throw Error("text content address hash should use %s, but instead uses %s", printHashType(htSHA256), printHashType(hashType)); - return TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), - }; + return TextHashMethod {}; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; HashType hashType = parseHashType_(); - return FixedOutputHash { - .method = method, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)), + return FixedOutputHashMethod { + .fileIngestionMethod = method, + .hashType = std::move(hashType), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto rest = rawCa; + + ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + + return std::visit( + overloaded { + [&](TextHashMethod thm) { + return ContentAddress(TextHash { + .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) + }); + }, + [&](FixedOutputHashMethod fohMethod) { + return ContentAddress(FixedOutputHash { + .method = fohMethod.fileIngestionMethod, + .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), + }); + }, + }, caMethod); +} + +ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +{ + std::string_view asPrefix {std::string{caMethod} + ":"}; + return parseContentAddressMethodPrefix(asPrefix); +} + +std::optional parseContentAddressOpt(std::string_view rawCaOpt) +{ + return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); }; -std::optional parseContentAddressOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional {} : parseContentAddress(rawCaOpt); -}; - -std::string renderContentAddress(std::optional ca) { +std::string renderContentAddress(std::optional ca) +{ return ca ? renderContentAddress(*ca) : ""; } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 22a039242..f6a6f5140 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -55,4 +55,23 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8adabf549..83f8968b0 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,7 +2,6 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" -#include "local-store.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -240,6 +239,23 @@ struct ClientSettings } }; +static void writeValidPathInfo( + ref store, + unsigned int clientVersion, + Sink & to, + std::shared_ptr info) +{ + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + writeStorePaths(*store, to, info->references); + to << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << renderContentAddress(info->ca); + } +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -350,47 +366,83 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - HashType hashAlgo; - std::string baseName; - FileIngestionMethod method; - { - bool fixed; - uint8_t recursive; - std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) - throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; - /* Compatibility hack. */ - if (!fixed) { - hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { + auto name = readString(from); + auto camStr = readString(from); + auto refs = readStorePaths(*store, from); + bool repairBool; + from >> repairBool; + auto repair = RepairFlag{repairBool}; + + logger->startWork(); + auto pathInfo = [&]() { + // NB: FramedSource must be out of scope before logger->stopWork(); + ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + FramedSource source(from); + // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. + return std::visit(overloaded { + [&](TextHashMethod &_) { + // We could stream this by changing Store + std::string contents = source.drain(); + auto path = store->addTextToStore(name, contents, refs, repair); + return store->queryPathInfo(path); + }, + [&](FixedOutputHashMethod &fohm) { + if (!refs.empty()) + throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + return store->queryPathInfo(path); + }, + }, contentAddressMethod); + }(); + logger->stopWork(); + + to << store->printStorePath(pathInfo->path); + writeValidPathInfo(store, clientVersion, to, pathInfo); + + } else { + HashType hashAlgo; + std::string baseName; + FileIngestionMethod method; + { + bool fixed; + uint8_t recursive; + std::string hashAlgoRaw; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + hashAlgoRaw = "sha256"; + method = FileIngestionMethod::Recursive; + } + hashAlgo = parseHashType(hashAlgoRaw); } - hashAlgo = parseHashType(hashAlgoRaw); + + StringSink saved; + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; + + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + // FIXME: try to stream directly from `from`. + StringSource dumpSource { *saved.s }; + auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + logger->stopWork(); + + to << store->printStorePath(path); } - - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - - logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); - logger->stopWork(); - - to << store->printStorePath(path); break; } @@ -675,15 +727,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } + writeValidPathInfo(store, clientVersion, to, info); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -749,59 +793,12 @@ static void performOp(TunnelLogger * logger, ref store, info.ultimate = false; if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { - - struct FramedSource : Source - { - Source & from; - bool eof = false; - std::vector pending; - size_t pos = 0; - - FramedSource(Source & from) : from(from) - { } - - ~FramedSource() - { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); - } - } - } - - size_t read(unsigned char * data, size_t len) override - { - if (eof) throw EndOfFile("reached end of FramedSource"); - - if (pos >= pending.size()) { - size_t len = readInt(from); - if (!len) { - eof = true; - return 0; - } - pending = std::vector(len); - pos = 0; - from(pending.data(), len); - } - - auto n = std::min(len, pending.size() - pos); - memcpy(data, pending.data() + pos, n); - pos += n; - return n; - } - }; - logger->startWork(); - { FramedSource source(from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } - logger->stopWork(); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 128832e60..49641c2ac 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 4149f8155..6241b5e00 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "compression.hh" #include "finally.hh" +#include "callback.hh" #ifdef ENABLE_S3 #include diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 491c664db..c5734852d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "abstract-setting-to-json.hh" #include #include @@ -41,6 +42,7 @@ Settings::Settings() { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; + allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { @@ -146,6 +148,12 @@ bool Settings::isWSL1() const string nixVersion = PACKAGE_VERSION; +NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { + {SandboxMode::smEnabled, true}, + {SandboxMode::smRelaxed, "relaxed"}, + {SandboxMode::smDisabled, false}, +}); + template<> void BaseSetting::set(const std::string & str) { if (str == "true") value = smEnabled; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 02721285a..ebcfa9d80 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,7 +2,6 @@ #include "types.hh" #include "config.hh" -#include "abstractsettingtojson.hh" #include "util.hh" #include @@ -881,6 +880,19 @@ public: Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; + + Setting allowSymlinkedStore{ + this, false, "allow-symlinked-store", + R"( + If set to `true`, Nix will stop complaining if the store directory + (typically /nix/store) contains symlink components. + + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems + occur if those builds are then deployed to machines where /nix/store + resolves to a different location from that of the build machine. You + can enable this setting if you are sure you're not going to do that. + )"}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index f4ab15a10..86be7c006 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e9478c1d5..5af75669a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" +#include "callback.hh" namespace nix { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c618203f0..c91f3fbf7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "callback.hh" #include #include @@ -109,7 +110,7 @@ LocalStore::LocalStore(const Params & params) } /* Ensure that the store and its parents are not symlinks. */ - if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + if (!settings.allowSymlinkedStore) { Path path = realStoreDir; struct stat st; while (path != "/") { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index da3981696..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "thread-pool.hh" #include "topo-sort.hh" - +#include "callback.hh" namespace nix { diff --git a/src/libstore/names.cc b/src/libstore/names.cc index d1c8a6101..41e28dc99 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,10 +1,18 @@ #include "names.hh" #include "util.hh" +#include + namespace nix { +struct Regex +{ + std::regex regex; +}; + + DrvName::DrvName() { name = ""; @@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0) } +DrvName::~DrvName() +{ } + + bool DrvName::matches(DrvName & n) { if (name != "*") { - if (!regex) regex = std::unique_ptr(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; + if (!regex) { + regex = std::make_unique(); + regex->regex = std::regex(name, std::regex::extended); + } + if (!std::regex_match(n.name, regex->regex)) return false; } if (version != "" && version != n.version) return false; return true; @@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; for (auto & i : opArgs) - result.push_back(DrvName(i)); + result.emplace_back(i); return result; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 00e14b8c7..bc62aac93 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -3,10 +3,11 @@ #include #include "types.hh" -#include namespace nix { +struct Regex; + struct DrvName { string fullName; @@ -16,10 +17,12 @@ struct DrvName DrvName(); DrvName(std::string_view s); + ~DrvName(); + bool matches(DrvName & n); private: - std::unique_ptr regex; + std::unique_ptr regex; }; typedef list DrvNames; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e92b94975..6f1f9769b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -10,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include #include @@ -306,6 +307,8 @@ struct ConnectionHandle std::rethrow_exception(ex); } } + + void withFramedSink(std::function fun); }; @@ -419,11 +422,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = readStorePaths(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - std::shared_ptr info; + std::shared_ptr info; { auto conn(getConnection()); conn->to << wopQueryPathInfo << printStorePath(path); @@ -439,17 +459,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - info = std::make_shared(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } + info = readValidPathInfo(conn, path); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -524,6 +534,90 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } +ref RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) +{ + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(caMethod); + writeStorePaths(*this, conn->to, references); + conn->to << repair; + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + } + }, caMethod); + auto path = parseStorePath(readString(conn->from)); + return queryPathInfo(path); + } +} + + +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; +} + + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -564,78 +658,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << repair << !checkSigs; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { - - conn->to.flush(); - - std::exception_ptr ex; - - struct FramedSink : BufferedSink - { - ConnectionHandle & conn; - std::exception_ptr & ex; - - FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) - { } - - ~FramedSink() - { - try { - conn->to << 0; - conn->to.flush(); - } catch (...) { - ignoreException(); - } - } - - void write(const unsigned char * data, size_t len) override - { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } - conn->to << len; - conn->to(data, len); - }; - }; - - /* Handle log messages / exceptions from the remote on a - separate thread. */ - std::thread stderrThread([&]() - { - try { - conn.processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink(conn, ex); + conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); - sink.flush(); - } - - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); - + }); } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { conn.processStderr(0, &source); } else { @@ -646,57 +671,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to - << wopAddToStore - << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } - - return parseStorePath(readString(conn->from)); -} - - StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); - - conn.processStderr(); - return parseStorePath(readString(conn->from)); + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } @@ -992,6 +971,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } +void ConnectionHandle::withFramedSink(std::function fun) +{ + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink((*this)->to, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + static RegisterStoreImplementation regStore; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 91c748006..ec04be985 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,13 +63,21 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + /* Add a content-addressable store path. `dump` will be drained. */ + ref addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair); + + /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -139,6 +147,8 @@ protected: virtual void narFromPath(const StorePath & path, Sink & sink) override; + ref readValidPathInfo(ConnectionHandle & conn, const StorePath & path); + private: std::atomic_bool failed{false}; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2d5077ed0..1bbc74db8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,9 +8,7 @@ #include "json.hh" #include "url.hh" #include "archive.hh" - -#include - +#include "callback.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d3f07dfc..591140874 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -449,7 +449,8 @@ public: /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == - false). */ + false). + `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 13cf8d4ab..b100d1550 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x118 +#define PROTOCOL_VERSION 0x119 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -18,7 +18,7 @@ typedef enum { wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, - wopAddTextToStore = 8, + wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore wopBuildPaths = 9, wopEnsurePath = 10, wopAddTempRoot = 11, diff --git a/src/libutil/abstractsettingtojson.hh b/src/libutil/abstract-setting-to-json.hh similarity index 100% rename from src/libutil/abstractsettingtojson.hh rename to src/libutil/abstract-setting-to-json.hh diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh new file mode 100644 index 000000000..ef31794be --- /dev/null +++ b/src/libutil/callback.hh @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +namespace nix { + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future to pass the value or + exception.) */ +template +class Callback +{ + std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 309d23b40..5e6a211df 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,6 @@ #include "config.hh" #include "args.hh" -#include "abstractsettingtojson.hh" +#include "abstract-setting-to-json.hh" #include diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041f..6e69bdce2 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include "ansicolor.hh" diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 00c945113..a469a1e73 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -93,7 +93,7 @@ void Source::operator () (unsigned char * data, size_t len) } -std::string Source::drain() +void Source::drainInto(Sink & sink) { std::string s; std::vector buf(8192); @@ -101,12 +101,19 @@ std::string Source::drain() size_t n; try { n = read(buf.data(), buf.size()); - s.append((char *) buf.data(), n); + sink(buf.data(), n); } catch (EndOfFile &) { break; } } - return s; +} + + +std::string Source::drain() +{ + StringSink s; + drainInto(s); + return *s.s; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6f4f4c855..b41e58f33 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -69,6 +69,8 @@ struct Source virtual bool good() { return true; } + void drainInto(Sink & sink); + std::string drain(); }; @@ -340,7 +342,7 @@ T readNum(Source & source) ((uint64_t) buf[6] << 48) | ((uint64_t) buf[7] << 56); - if (n > std::numeric_limits::max()) + if (n > (uint64_t)std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; @@ -404,4 +406,93 @@ struct StreamToSourceAdapter : Source }; +/* A source that reads a distinct format of concatenated chunks back into its + logical form, in order to guarantee a known state to the original stream, + even in the event of errors. + + Use with FramedSink, which also allows the logical stream to be terminated + in the event of an exception. +*/ +struct FramedSource : Source +{ + Source & from; + bool eof = false; + std::vector pending; + size_t pos = 0; + + FramedSource(Source & from) : from(from) + { } + + ~FramedSource() + { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } + } + } + + size_t read(unsigned char * data, size_t len) override + { + if (eof) throw EndOfFile("reached end of FramedSource"); + + if (pos >= pending.size()) { + size_t len = readInt(from); + if (!len) { + eof = true; + return 0; + } + pending = std::vector(len); + pos = 0; + from(pending.data(), len); + } + + auto n = std::min(len, pending.size() - pos); + memcpy(data, pending.data() + pos, n); + pos += n; + return n; + } +}; + +/* Write as chunks in the format expected by FramedSource. + + The exception_ptr reference can be used to terminate the stream when you + detect that an error has occurred on the remote end. +*/ +struct FramedSink : nix::BufferedSink +{ + BufferedSink & to; + std::exception_ptr & ex; + + FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex) + { } + + ~FramedSink() + { + try { + to << 0; + to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + to << len; + to(data, len); + }; +}; + + } diff --git a/src/libutil/split.hh b/src/libutil/split.hh index d19d7d8ed..87a23b13e 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -9,7 +9,7 @@ namespace nix { // If `separator` is found, we return the portion of the string before the // separator, and modify the string argument to contain only the part after the -// separator. Otherwise, wer return `std::nullopt`, and we leave the argument +// separator. Otherwise, we return `std::nullopt`, and we leave the argument // string alone. static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh new file mode 100644 index 000000000..64e06cfbc --- /dev/null +++ b/src/libutil/url-parts.hh @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace nix { + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+.-]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 88c09eef9..c1bab866c 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,4 +1,5 @@ #include "url.hh" +#include "url-parts.hh" #include "util.hh" namespace nix { diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 1f716ba10..6e77142e3 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -2,8 +2,6 @@ #include "error.hh" -#include - namespace nix { struct ParsedURL @@ -29,40 +27,4 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -// URI stuff. -const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+.-]+)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; -const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; -const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; -const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; -const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; -const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; -const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; -const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; -const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; - -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -extern std::regex refRegex; - -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition -const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; -extern std::regex badGitRefRegex; - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -extern std::regex revRegex; - -// A ref or revision, or a ref followed by a revision. -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 082e26375..b8e201203 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,13 +12,9 @@ #include #include -#include -#include #include #include #include -#include -#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -480,43 +476,8 @@ std::optional get(const T & map, const typename T::key_ } -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ template -class Callback -{ - std::function)> fun; - std::atomic_flag done = ATOMIC_FLAG_INIT; - -public: - - Callback(std::function)> fun) : fun(fun) { } - - Callback(Callback && callback) : fun(std::move(callback.fun)) - { - auto prev = callback.done.test_and_set(); - if (prev) done.test_and_set(); - } - - void operator()(T && t) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_value(std::move(t)); - fun(promise.get_future()); - } - - void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_exception(exc); - fun(promise.get_future()); - } -}; +class Callback; /* Start a thread that handles various signals. Also block those signals diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e5a433ac0..3e7c453fb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) - selectors.push_back(DrvName("*")); + selectors.emplace_back("*"); DrvInfos elems; set done;