From b2be6fed8600ee48c05cc9643c101d5eab4a5727 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Apr 2024 16:00:52 +0200 Subject: [PATCH 001/243] Improve support for subflakes Subflakes are flakes in the same tree, accessed in flake inputs via relative paths (e.g. `inputs.foo.url = "path:./subdir"`). Previously these didn't work very well because they would be separately copied to the store, which is inefficient and makes references to parent directories tricky or impossible. Furthermore, they had their own NAR hash in the lock file, which is superfluous since the parent is already locked. Now subflakes are accessed via the accessor of the calling flake. This avoids the unnecessary copy and makes it possible for subflakes to depend on flakes in a parent directory (so long as they're in the same tree). Lock file nodes for relative flake inputs now have a new `parent` field: { "locked": { "path": "./subdir", "type": "path" }, "original": { "path": "./subdir", "type": "path" }, "parent": [ "foo", "bar" ] } which denotes that `./subdir` is to be interpreted relative to the directory of the `bar` input of the `foo` input of the root flake. Extracted from the lazy-trees branch. --- src/libexpr/flake/call-flake.nix | 7 + src/libexpr/flake/flake.cc | 160 +++++++++++++++------- src/libexpr/flake/flakeref.cc | 68 +++++---- src/libexpr/flake/flakeref.hh | 6 +- src/libexpr/flake/lockfile.cc | 9 +- src/libexpr/flake/lockfile.hh | 12 +- src/libexpr/primops/fetchTree.cc | 5 +- src/libfetchers/fetchers.cc | 6 + src/libfetchers/fetchers.hh | 14 +- src/libfetchers/path.cc | 25 +--- src/nix/flake.md | 25 +++- tests/functional/flakes/follow-paths.sh | 25 +++- tests/functional/flakes/relative-paths.sh | 89 ++++++++++++ tests/functional/local.mk | 1 + 14 files changed, 332 insertions(+), 120 deletions(-) create mode 100644 tests/functional/flakes/relative-paths.sh diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index a411564df..43ecb7f15 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -38,10 +38,17 @@ let (key: node: let + parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + sourceInfo = if overrides ? ${key} then overrides.${key}.sourceInfo + else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" + then + parentNode.sourceInfo // { + outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + } else # FIXME: remove obsolete node.info. fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..d4cabe68f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -93,12 +93,17 @@ static void expectType(EvalState & state, ValueType type, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath); + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath); -static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) +static FlakeInput parseFlakeInput( + EvalState & state, + const std::string & inputName, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -122,7 +127,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -173,7 +178,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(*url, {}, true, input.isFlake, true); } if (!input.follows && !input.ref) @@ -183,8 +188,10 @@ static FlakeInput parseFlakeInput(EvalState & state, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { std::map inputs; @@ -196,7 +203,6 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - baseDir, lockRootPath)); } @@ -232,7 +238,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); auto sOutputs = state.symbols.create("outputs"); @@ -366,13 +372,29 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - std::map overrides; + struct Override + { + FlakeInput input; + SourcePath sourcePath; + std::optional parentPath; // FIXME: rename to inputPathPrefix? + }; + + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { - overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + overrides.emplace( + i.first, + Override { + .input = FlakeInput { .ref = i.second }, + /* Note: any relative overrides + (e.g. `--override-input B/C "path:./foo/bar"`) + are interpreted relative to the top-level + flake. */ + .sourcePath = flake.path, + }); explicitCliOverrides.insert(i.first); } @@ -386,7 +408,7 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + const SourcePath & sourcePath, bool trustLock)> computeLocks; @@ -402,7 +424,8 @@ LockedFlake lockFlake( copied. */ std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + /* The source path of this node's flake. */ + const SourcePath & sourcePath, bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -414,7 +437,12 @@ LockedFlake lockFlake( auto inputPath(inputPathPrefix); inputPath.push_back(id); inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); + overrides.emplace(inputPath, + Override { + .input = inputOverride, + .sourcePath = sourcePath, + .parentPath = inputPathPrefix // FIXME: should this be inputPath? + }); } } @@ -446,13 +474,18 @@ LockedFlake lockFlake( auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); bool hasCliOverride = explicitCliOverrides.contains(inputPath); - if (hasOverride) { + if (hasOverride) overridesUsed.insert(inputPath); - // Respect the “flakeness” of the input even if we - // override it - i->second.isFlake = input2.isFlake; - } - auto & input = hasOverride ? i->second : input2; + auto input = hasOverride ? i->second.input : input2; + + /* Resolve relative 'path:' inputs relative to + the source path of the overrider. */ + auto overridenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; + + /* Respect the "flakeness" of the input even if we + override it. */ + if (hasOverride) + input.isFlake = input2.isFlake; /* Resolve 'follows' later (since it may refer to an input path we haven't processed yet. */ @@ -468,6 +501,33 @@ LockedFlake lockFlake( assert(input.ref); + auto overridenParentPath = + input.ref->input.isRelative() + ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + : std::nullopt; + + auto resolveRelativePath = [&]() -> std::optional + { + if (auto relativePath = input.ref->input.isRelative()) { + return SourcePath { + overridenSourcePath.accessor, + CanonPath(*relativePath, overridenSourcePath.path.parent().value()) + }; + } else + return std::nullopt; + }; + + /* Get the input flake, resolve 'path:./...' + flakerefs relative to the parent flake. */ + auto getInputFlake = [&]() + { + if (auto resolvedPath = resolveRelativePath()) { + return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath); + } else { + return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath); + } + }; + /* Do we have an entry in the existing lock file? And the input is not in updateInputs? */ std::shared_ptr oldLock; @@ -481,6 +541,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref + && oldLock->parentPath == overridenParentPath && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -489,7 +550,10 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->parentPath); node->inputs.insert_or_assign(id, childNode); @@ -541,11 +605,15 @@ LockedFlake lockFlake( } if (mustRefetch) { - auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); + auto inputFlake = getInputFlake(); nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); + // FIXME: sourcePath is wrong here, we + // should pass a lambda that lazily + // fetches the parent flake if needed + // (i.e. getInputFlake()). + computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } } else { @@ -553,7 +621,9 @@ LockedFlake lockFlake( this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + if (!lockFlags.allowUnlocked + && !input.ref->input.isLocked() + && !input.ref->input.isRelative()) throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); /* Note: in case of an --override-input, we use @@ -566,17 +636,9 @@ LockedFlake lockFlake( auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { - Path localPath = parentPath; - FlakeRef localRef = *input.ref; + auto inputFlake = getInputFlake(); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if (localRef.input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); - - auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - - auto childNode = make_ref(inputFlake.lockedRef, ref); + auto childNode = make_ref(inputFlake.lockedRef, ref, true, overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -598,17 +660,26 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast(oldLock) : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, - localPath, + inputFlake.path, false); } else { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, useRegistries, flakeCache); + auto [path, lockedRef] = [&]() -> std::tuple + { + // Handle non-flake 'path:./...' inputs. + if (auto resolvedPath = resolveRelativePath()) { + return {*resolvedPath, *input.ref}; + } else { + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, useRegistries, flakeCache); + return {state.rootPath(state.store->toRealPath(storePath)), lockedRef}; + } + }(); - auto childNode = make_ref(lockedRef, ref, false); + auto childNode = make_ref(lockedRef, ref, false, overridenParentPath); - nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); + nodePaths.emplace(childNode, path); node->inputs.insert_or_assign(id, childNode); } @@ -621,9 +692,6 @@ LockedFlake lockFlake( } }; - // Bring in the current ref for relative path resolution if we have it - auto parentPath = flake.path.parent().path.abs(); - nodePaths.emplace(newLockFile.root, flake.path.parent()); computeLocks( @@ -632,7 +700,7 @@ LockedFlake lockFlake( {}, lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), {}, - parentPath, + flake.path, false); for (auto & i : lockFlags.inputOverrides) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6e4aad64d..bac237a2a 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -51,9 +51,10 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -69,11 +70,25 @@ std::optional maybeParseFlakeRef( } } +static std::pair fromParsedURL( + ParsedURL && parsedURL, + bool isFlake) +{ + auto dir = getOr(parsedURL.query, "dir", ""); + parsedURL.query.erase("dir"); + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), dir), fragment); +} + std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { std::string path = url; std::string fragment = ""; @@ -90,7 +105,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); } if (baseDir) { @@ -154,6 +169,7 @@ std::pair parsePathFlakeRefWithFragment( .authority = "", .path = flakeRoot, .query = query, + .fragment = fragment, }; if (subdir != "") { @@ -165,9 +181,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); + return fromParsedURL(std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -176,25 +190,30 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!hasPrefix(path, "/")) + if (!allowRelative && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); } fetchers::Attrs attrs; attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); -}; - + return fromParsedURL({ + .url = path, + .base = path, + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); +} /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ std::optional> parseFlakeIdRef( const std::string & url, - bool isFlake -) + bool isFlake) { std::smatch match; @@ -223,32 +242,21 @@ std::optional> parseFlakeIdRef( std::optional> parseURLFlakeRef( const std::string & url, const std::optional & baseDir, - bool isFlake -) + bool isFlake) { - ParsedURL parsedURL; try { - parsedURL = parseURL(url); + return fromParsedURL(parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } - - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = fetchers::Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); } std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { using namespace fetchers; @@ -259,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 04c812ed0..ea6c4e4d7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -75,7 +75,8 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -90,7 +91,8 @@ std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index d252214dd..2884ca262 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -36,8 +36,9 @@ LockedNode::LockedNode(const nlohmann::json & json) : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , parentPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { - if (!lockedRef.input.isLocked()) + if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) throw Error("lock file contains unlocked input '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -184,6 +185,8 @@ std::pair LockFile::toJSON() const n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->parentPath) + n["parent"] = *lockedNode->parentPath; } nodes[key] = std::move(n); @@ -230,7 +233,9 @@ std::optional LockFile::isUnlocked() const for (auto & i : nodes) { if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); - if (node && !node->lockedRef.input.isLocked()) + if (node + && !node->lockedRef.input.isLocked() + && !node->lockedRef.input.isRelative()) return node->lockedRef; } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 7e62e6d09..aad805baf 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -38,11 +38,19 @@ struct LockedNode : Node FlakeRef lockedRef, originalRef; bool isFlake = true; + /* The node relative to which relative source paths + (e.g. 'path:../foo') are interpreted. */ + std::optional parentPath; + LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + bool isFlake = true, + std::optional parentPath = {}) + : lockedRef(lockedRef) + , originalRef(originalRef) + , isFlake(isFlake) + , parentPath(parentPath) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e27f30512..1c5cd4ec2 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,9 +31,8 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); + if (auto narHash = input.getNarHash()) + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..73cb4fea3 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -141,6 +141,12 @@ bool Input::isLocked() const return scheme && scheme->isLocked(*this); } +std::optional Input::isRelative() const +{ + assert(scheme); + return scheme->isRelative(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f..66cac7064 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -31,11 +31,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; - /** - * path of the parent of this input, used for relative path resolution - */ - std::optional parent; - public: /** * Create an `Input` from a URL. @@ -73,6 +68,12 @@ public: */ bool isLocked() const; + /** + * Only for relative path flakes, i.e. 'path:./foo', returns the + * relative path, i.e. './foo'. + */ + std::optional isRelative() const; + bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -220,6 +221,9 @@ struct InputScheme * if there is a mismatch. */ virtual void checkLocks(const Input & specified, const Input & final) const; + + virtual std::optional isRelative(const Input & input) const + { return std::nullopt; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d559..62500bbcb 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -116,31 +116,14 @@ struct PathInputScheme : InputScheme std::pair, Input> getAccessor(ref store, const Input & _input) const override { Input input(_input); - std::string absPath; auto path = getStrAttr(input.attrs, "path"); - if (path[0] != '/') { - if (!input.parent) - throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); + auto absPath = getAbsPath(input); - auto parent = canonPath(*input.parent); - - // the path isn't relative, prefix it - absPath = nix::absPath(path, parent); - - // for security, ensure that if the parent is a store path, it's inside it - if (store->isInStore(parent)) { - auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isDirOrInDir(absPath, storePath)) - throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); - } - } else - absPath = path; - - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath); + auto storePath = store->maybeParseStorePath(absPath.abs()); if (storePath) store->addTempRoot(*storePath); @@ -149,7 +132,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } diff --git a/src/nix/flake.md b/src/nix/flake.md index 661dd2f73..e60abe5fc 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -195,18 +195,29 @@ Currently the `type` attribute can be one of the following: If the flake at *path* is not inside a git repository, the `path:` prefix is implied and can be omitted. - *path* generally must be an absolute path. However, on the command - line, it can be a relative path (e.g. `.` or `./foo`) which is - interpreted as relative to the current directory. In this case, it - must start with `.` to avoid ambiguity with registry lookups - (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative - path). + If *path* is a relative path (i.e. if it does not start with `/`), + it is interpreted as follows: + + - If *path* is a command line argument, it is interpreted relative + to the current directory. + + - If *path* is used in a `flake.nix`, it is interpreted relative to + the directory containing that `flake.nix`. However, the resolved + path must be in the same tree. For instance, a `flake.nix` in the + root of a tree can use `path:./foo` to access the flake in + subdirectory `foo`, but `path:../bar` is illegal. + + Note that if you omit `path:`, relative paths must start with `.` to + avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry + lookup; `./nixpkgs` is a relative path). For example, these are valid path flake references: * `path:/home/user/sub/dir` * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) - * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `path:sub/dir` + * `./sub/dir` + * `path:../parent` * `git`: Git repositories. The location of the repository is specified by the attribute `url`. diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 1afd91bd2..35f52f3ba 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -115,7 +115,7 @@ nix flake lock $flakeFollowsA [[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '"foobar"' ]] jq -r -c '.nodes | keys | .[]' $flakeFollowsA/flake.lock | grep "^foobar$" -# Ensure a relative path is not allowed to go outside the store path +# Check that path: inputs cannot escape from their root. cat > $flakeFollowsA/flake.nix <&1 | grep 'points outside' +expect 1 nix flake lock $flakeFollowsA 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' +expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' + +# Test relative non-flake inputs. +cat > $flakeFollowsA/flake.nix < $flakeFollowsA/foo.nix + +git -C $flakeFollowsA add flake.nix foo.nix + +nix flake lock $flakeFollowsA + +[[ $(nix eval --json $flakeFollowsA#e) = 123 ]] # Non-existant follows should print a warning. cat >$flakeFollowsA/flake.nix < $rootFlake/flake.nix < $subflake0/flake.nix < $subflake1/flake.nix < $subflake2/flake.nix < $rootFlake/flake.nix < Date: Fri, 17 May 2024 16:38:01 +0200 Subject: [PATCH 002/243] call-flake.nix: Fix relative path resolution `parentNode.sourceInfo.outPath` does not include the subdir of the parent flake, while `parentNode.outPath` does. So we need to use the latter. --- src/libexpr/flake/call-flake.nix | 2 +- tests/functional/flakes/relative-paths.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 43ecb7f15..38dc74ba1 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -47,7 +47,7 @@ let else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + outPath = parentNode.outPath + ("/" + node.locked.path); } else # FIXME: remove obsolete node.info. diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index cecda44d4..38987d6af 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -63,6 +63,10 @@ git -C $rootFlake add flake.nix sub2/flake.nix [[ $(nix eval $subflake2#y) = 15 ]] +# Make sure that this still works after commiting the lock file. +git -C $rootFlake add sub2/flake.lock +[[ $(nix eval $subflake2#y) = 15 ]] + # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" $subflake2/flake.lock) (! grep "$NIX_STORE_DIR" $subflake2/flake.lock) From 3180671cabeb6a6010057770731e12761ed5666c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 May 2024 19:49:40 +0200 Subject: [PATCH 003/243] Allow the 'url' flake input attribute to be a path literal https://github.com/NixOS/nix/pull/10089#issuecomment-1978133326 --- src/libexpr/flake/flake.cc | 34 +++++++++++++++++------ tests/functional/flakes/relative-paths.sh | 4 +-- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index d4cabe68f..1401f5dda 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -96,14 +96,16 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath); + InputPath lockRootPath, + const SourcePath & flakeDir); static FlakeInput parseFlakeInput( EvalState & state, const std::string & inputName, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { expectType(state, nAttrs, *value, pos); @@ -120,14 +122,25 @@ static FlakeInput parseFlakeInput( for (auto & attr : *value->attrs()) { try { if (attr.name == sUrl) { - expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string_view(); + forceTrivialValue(state, *attr.value, pos); + if (attr.value->type() == nString) + url = attr.value->string_view(); + else if (attr.value->type() == nPath) { + auto path = attr.value->path(); + if (path.accessor != flakeDir.accessor) + throw Error("input path '%s' at %s must be in the same source tree as %s", + path, state.positions[attr.pos], flakeDir); + url = "path:" + flakeDir.path.makeRelative(path.path); + } + else + throw Error("expected a string or a path but got %s at %s", + showType(attr.value->type()), state.positions[attr.pos]); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath, flakeDir); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -191,7 +204,8 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { std::map inputs; @@ -203,7 +217,8 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - lockRootPath)); + lockRootPath, + flakeDir)); } return inputs; @@ -217,7 +232,8 @@ static Flake readFlake( const SourcePath & rootDir, const InputPath & lockRootPath) { - auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; + auto flakeDir = rootDir / CanonPath(resolvedRef.subdir); + auto flakePath = flakeDir / "flake.nix"; // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true. Value vInfo; @@ -238,7 +254,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath, flakeDir); auto sOutputs = state.symbols.create("outputs"); diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 38987d6af..3e4c97cc4 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -12,7 +12,7 @@ mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 cat > $rootFlake/flake.nix < $subflake2/flake.nix < Date: Mon, 16 Sep 2024 14:11:08 +0200 Subject: [PATCH 004/243] shellcheck --- tests/functional/flakes/relative-paths.sh | 54 ++++++++++++----------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 3e4c97cc4..e72339b7f 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + source ./common.sh requireGit -rootFlake=$TEST_ROOT/flake1 -subflake0=$rootFlake/sub0 -subflake1=$rootFlake/sub1 -subflake2=$rootFlake/sub2 +rootFlake="$TEST_ROOT/flake1" +subflake0="$rootFlake/sub0" +subflake1="$rootFlake/sub1" +subflake2="$rootFlake/sub2" -rm -rf $rootFlake -mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 +rm -rf "$rootFlake" +mkdir -p "$rootFlake" "$subflake0" "$subflake1" "$subflake2" -cat > $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < $subflake0/flake.nix < "$subflake0/flake.nix" < $subflake0/flake.nix < $subflake1/flake.nix < "$subflake1/flake.nix" < $subflake1/flake.nix < $subflake2/flake.nix < "$subflake2/flake.nix" < $subflake2/flake.nix < $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < Date: Mon, 16 Sep 2024 14:52:23 +0200 Subject: [PATCH 005/243] parentPath -> parentInputPath --- src/libflake/flake/flake.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 632968f50..461ac0a90 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -404,7 +404,7 @@ LockedFlake lockFlake( { FlakeInput input; SourcePath sourcePath; - std::optional parentPath; // FIXME: rename to inputPathPrefix? + std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; std::map overrides; @@ -469,7 +469,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? }); } } @@ -531,7 +531,7 @@ LockedFlake lockFlake( auto overridenParentPath = input.ref->input.isRelative() - ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + ? std::optional(hasOverride ? i->second.parentInputPath : inputPathPrefix) : std::nullopt; auto resolveRelativePath = [&]() -> std::optional From f2063255a40d534035b706235752a5c0a09c6d15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Sep 2024 16:29:43 +0200 Subject: [PATCH 006/243] tests/functional/flakes/relative-paths.sh: Fix build failure in hydraJobs.tests.functional_user --- tests/functional/flakes/relative-paths.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index e72339b7f..9b93da9c1 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -71,7 +71,9 @@ git -C "$rootFlake" add sub2/flake.lock # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") -(! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +if ! isTestOnNixOS; then + (! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +fi (! grep narHash "$subflake2/flake.lock") # Test circular relative path flakes. FIXME: doesn't work at the moment. From 76f75e76915329c4f3502abcbea1df8e3ee658c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 15:28:49 +0200 Subject: [PATCH 007/243] nix copy: Add --profile flag This allows `nix copy` to atomically copy a store path and point a profile to it, without the risk that the store path might be GC'ed in between. This is useful for instance when deploying a new NixOS system profile from a remote store. --- src/libcmd/command.cc | 35 ++++++++++++++++++++++------------- src/libcmd/command.hh | 12 ++++++++---- src/nix/build.cc | 2 +- src/nix/copy.cc | 8 +++++--- src/nix/copy.md | 9 +++++++++ src/nix/develop.cc | 2 +- src/nix/profile.cc | 6 +++--- src/nix/realisation.cc | 2 +- tests/functional/zstd.sh | 4 +++- 9 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6d8bfc19b..e38f982d8 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -179,30 +179,34 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive) void BuiltPathsCommand::run(ref store, Installables && installables) { - BuiltPaths paths; + BuiltPaths rootPaths, allPaths; + if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.emplace_back(BuiltPath::Opaque{p}); + rootPaths.emplace_back(BuiltPath::Opaque{p}); + allPaths = rootPaths; } else { - paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); + rootPaths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); + allPaths = rootPaths; + if (recursive) { // XXX: This only computes the store path closure, ignoring // intermediate realisations StorePathSet pathsRoots, pathsClosure; - for (auto & root : paths) { + for (auto & root : rootPaths) { auto rootFromThis = root.outPaths(); pathsRoots.insert(rootFromThis.begin(), rootFromThis.end()); } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.emplace_back(BuiltPath::Opaque{path}); + allPaths.emplace_back(BuiltPath::Opaque{path}); } } - run(store, std::move(paths)); + run(store, std::move(allPaths), std::move(rootPaths)); } StorePathsCommand::StorePathsCommand(bool recursive) @@ -210,10 +214,10 @@ StorePathsCommand::StorePathsCommand(bool recursive) { } -void StorePathsCommand::run(ref store, BuiltPaths && paths) +void StorePathsCommand::run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) { StorePathSet storePaths; - for (auto & builtPath : paths) + for (auto & builtPath : allPaths) for (auto & p : builtPath.outPaths()) storePaths.insert(p); @@ -242,17 +246,21 @@ MixProfile::MixProfile() }); } -void MixProfile::updateProfile(const StorePath & storePath) +void MixProfile::updateProfile( + const StorePath & storePath, + ref store_) { if (!profile) return; - auto store = getStore().dynamic_pointer_cast(); + auto store = store_.dynamic_pointer_cast(); if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, createGeneration(*store, profile2, storePath)); } -void MixProfile::updateProfile(const BuiltPaths & buildables) +void MixProfile::updateProfile( + const BuiltPaths & buildables, + ref store) { if (!profile) return; @@ -274,7 +282,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) if (result.size() != 1) throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(result[0]); + updateProfile(result[0], store); } MixDefaultProfile::MixDefaultProfile() @@ -308,7 +316,8 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) }); } -void MixEnvironment::setEnviron() { +void MixEnvironment::setEnviron() +{ if (ignoreEnvironment) { if (!unset.empty()) throw UsageError("--unset does not make sense with --ignore-environment"); diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 4a72627ed..8ada78782 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -238,7 +238,7 @@ public: BuiltPathsCommand(bool recursive = false); - virtual void run(ref store, BuiltPaths && paths) = 0; + virtual void run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) = 0; void run(ref store, Installables && installables) override; @@ -251,7 +251,7 @@ struct StorePathsCommand : public BuiltPathsCommand virtual void run(ref store, StorePaths && storePaths) = 0; - void run(ref store, BuiltPaths && paths) override; + void run(ref store, BuiltPaths && allPaths, BuiltPaths && rootPaths) override; }; /** @@ -301,11 +301,15 @@ struct MixProfile : virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile(const StorePath & storePath); + void updateProfile( + const StorePath & storePath, + ref store); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const BuiltPaths & buildables); + void updateProfile( + const BuiltPaths & buildables, + ref store); }; struct MixDefaultProfile : MixProfile diff --git a/src/nix/build.cc b/src/nix/build.cc index da9132d02..cffbeccb6 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile BuiltPaths buildables2; for (auto & b : buildables) buildables2.push_back(b.path); - updateProfile(buildables2); + updateProfile(buildables2, store); } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 151d28277..c058a7446 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -4,7 +4,7 @@ using namespace nix; -struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand +struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile { CheckSigsFlag checkSigs = CheckSigs; @@ -43,19 +43,21 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand Category category() override { return catSecondary; } - void run(ref srcStore, BuiltPaths && paths) override + void run(ref srcStore, BuiltPaths && allPaths, BuiltPaths && rootPaths) override { auto dstStore = getDstStore(); RealisedPath::Set stuffToCopy; - for (auto & builtPath : paths) { + for (auto & builtPath : allPaths) { auto theseRealisations = builtPath.toRealisedPaths(*srcStore); stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); } copyPaths( *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); + + updateProfile(rootPaths, dstStore); } }; diff --git a/src/nix/copy.md b/src/nix/copy.md index 6ab7cdee3..813050fcb 100644 --- a/src/nix/copy.md +++ b/src/nix/copy.md @@ -55,6 +55,15 @@ R""( # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs ``` +* Update the NixOS system profile to point to a closure copied from a + remote machine: + + ```console + # nix copy --from ssh://server \ + --profile /nix/var/nix/profiles/system \ + /nix/store/r14v3km89zm3prwsa521fab5kgzvfbw4-nixos-system-foobar-24.05.20240925.759537f + ``` + # Description `nix copy` copies store path closures between two Nix stores. The diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c7a733025..ca9f6a4af 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile auto strPath = store->printStorePath(shellOutPath); - updateProfile(shellOutPath); + updateProfile(shellOutPath, store); debug("reading environment file '%s'", strPath); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 324fd6330..ae1f3e6e9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } try { - updateProfile(manifest.build(store)); + updateProfile(manifest.build(store), store); } catch (BuildEnvFileConflictError & conflictError) { // FIXME use C++20 std::ranges once macOS has it // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 @@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - updateProfile(newManifest.build(store)); + updateProfile(newManifest.build(store), store); } }; @@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf builtPaths.find(&*installable)->second.first); } - updateProfile(manifest.build(store)); + updateProfile(manifest.build(store), store); } }; diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index e1f231222..a386d98ea 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -36,7 +36,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON Category category() override { return catSecondary; } - void run(ref store, BuiltPaths && paths) override + void run(ref store, BuiltPaths && paths, BuiltPaths && rootPaths) override { experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index 90fe58539..cdc30c66e 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -18,7 +18,9 @@ HASH=$(nix hash path $outPath) clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs +nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile + +[[ -e $TEST_ROOT/profile ]] if ls $cacheDir/nar/*.zst &> /dev/null; then echo "files do exist" From 43ad8c5eb2bdf4995467595f83efd6d9a747b440 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 15:36:21 +0200 Subject: [PATCH 008/243] Make getDstStore() a virtual method in StoreCommand --- src/libcmd/command.cc | 12 ++++-------- src/libcmd/command.hh | 23 ++++++++++++++++------- src/nix/build.cc | 2 +- src/nix/copy.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/profile.cc | 6 +++--- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index e38f982d8..0fb4f5680 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -246,21 +246,17 @@ MixProfile::MixProfile() }); } -void MixProfile::updateProfile( - const StorePath & storePath, - ref store_) +void MixProfile::updateProfile(const StorePath & storePath) { if (!profile) return; - auto store = store_.dynamic_pointer_cast(); + auto store = getDstStore().dynamic_pointer_cast(); if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, createGeneration(*store, profile2, storePath)); } -void MixProfile::updateProfile( - const BuiltPaths & buildables, - ref store) +void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; @@ -282,7 +278,7 @@ void MixProfile::updateProfile( if (result.size() != 1) throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(result[0], store); + updateProfile(result[0]); } MixDefaultProfile::MixDefaultProfile() diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 8ada78782..73aaab741 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -45,7 +45,20 @@ struct StoreCommand : virtual Command { StoreCommand(); void run() override; + + /** + * Return the default Nix store. + */ ref getStore(); + + /** + * Return the destination Nix store. + */ + virtual ref getDstStore() + { + return getStore(); + } + virtual ref createStore(); /** * Main entry point, with a `Store` provided @@ -68,7 +81,7 @@ struct CopyCommand : virtual StoreCommand ref createStore() override; - ref getDstStore(); + ref getDstStore() override; }; /** @@ -301,15 +314,11 @@ struct MixProfile : virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile( - const StorePath & storePath, - ref store); + void updateProfile(const StorePath & storePath); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile( - const BuiltPaths & buildables, - ref store); + void updateProfile(const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/nix/build.cc b/src/nix/build.cc index cffbeccb6..da9132d02 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -161,7 +161,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile BuiltPaths buildables2; for (auto & b : buildables) buildables2.push_back(b.path); - updateProfile(buildables2, store); + updateProfile(buildables2); } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c058a7446..60a64723c 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -57,7 +57,7 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile copyPaths( *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); - updateProfile(rootPaths, dstStore); + updateProfile(rootPaths); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ca9f6a4af..c7a733025 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -502,7 +502,7 @@ struct Common : InstallableCommand, MixProfile auto strPath = store->printStorePath(shellOutPath); - updateProfile(shellOutPath, store); + updateProfile(shellOutPath); debug("reading environment file '%s'", strPath); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ae1f3e6e9..324fd6330 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -424,7 +424,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } try { - updateProfile(manifest.build(store), store); + updateProfile(manifest.build(store)); } catch (BuildEnvFileConflictError & conflictError) { // FIXME use C++20 std::ranges once macOS has it // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 @@ -669,7 +669,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - updateProfile(newManifest.build(store), store); + updateProfile(newManifest.build(store)); } }; @@ -779,7 +779,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf builtPaths.find(&*installable)->second.first); } - updateProfile(manifest.build(store), store); + updateProfile(manifest.build(store)); } }; From 7f6d006bebfb56b2967165c7f2e8c940abafac24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 16:35:53 +0200 Subject: [PATCH 009/243] nix copy: Add --out-link --- src/libcmd/command.cc | 25 +++++++++++++++++++++++++ src/libcmd/command.hh | 10 ++++++++++ src/libcmd/installables.cc | 8 ++++++++ src/libcmd/installables.hh | 2 ++ src/nix/build.cc | 25 +------------------------ src/nix/copy.cc | 18 ++++++++++++++++++ tests/functional/zstd.sh | 3 ++- 7 files changed, 66 insertions(+), 25 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0fb4f5680..8053e1da6 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -334,4 +334,29 @@ void MixEnvironment::setEnviron() } } +void createOutLinks( + const std::filesystem::path & outLink, + const BuiltPaths & buildables, + LocalFSStore & store) +{ + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + store.addPermRoot(bo.path, absPath(symlink.string())); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store.addPermRoot(output.second, absPath(symlink.string())); + } + }, + }, buildable.raw()); + } +} + } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 73aaab741..e9fcf3df9 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -18,6 +18,7 @@ extern char * * savedArgv; class EvalState; struct Pos; class Store; +class LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; @@ -367,4 +368,13 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); +/** + * Create symlinks prefixed by `outLink` to the store paths in + * `buildables`. + */ +void createOutLinks( + const std::filesystem::path & outLink, + const BuiltPaths & buildables, + LocalFSStore & store); + } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f9d6d8ce8..86d26f6c4 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -917,4 +917,12 @@ void BuiltPathsCommand::applyDefaultInstallables(std::vector & rawI rawInstallables.push_back("."); } +BuiltPaths toBuiltPaths(const std::vector & builtPathsWithResult) +{ + BuiltPaths res; + for (auto & i : builtPathsWithResult) + res.push_back(i.path); + return res; +} + } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index bf5759230..c995c3019 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -86,6 +86,8 @@ struct BuiltPathWithResult std::optional result; }; +BuiltPaths toBuiltPaths(const std::vector & builtPathsWithResult); + /** * Shorthand, for less typing and helping us keep the choice of * collection in sync. diff --git a/src/nix/build.cc b/src/nix/build.cc index da9132d02..3569b0cde 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -42,29 +42,6 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector& buildables, LocalFSStore& store2) -{ - for (const auto & [_i, buildable] : enumerate(buildables)) { - auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - store2.addPermRoot(bo.path, absPath(symlink.string())); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2.addPermRoot(output.second, absPath(symlink.string())); - } - }, - }, buildable.path.raw()); - } -} - struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; @@ -140,7 +117,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - createOutLinks(outLink, buildables, *store2); + createOutLinks(outLink, toBuiltPaths(buildables), *store2); if (printOutputPaths) { stopProgressBar(); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 60a64723c..399a6c0fd 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -1,11 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" using namespace nix; struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile { + std::optional outLink; CheckSigsFlag checkSigs = CheckSigs; SubstituteFlag substitute = NoSubstitute; @@ -13,6 +15,15 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile CmdCopy() : BuiltPathsCommand(true) { + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Create symlinks prefixed with *path* to the top-level store paths fetched from the source store.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + addFlag({ .longName = "no-check-sigs", .description = "Do not require that paths are signed by trusted keys.", @@ -58,6 +69,13 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); updateProfile(rootPaths); + + if (outLink) { + if (auto store2 = dstStore.dynamic_pointer_cast()) + createOutLinks(*outLink, rootPaths, *store2); + else + throw Error("'--out-link' is not supported for this Nix store"); + } } }; diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index cdc30c66e..1eb1b5343 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -18,9 +18,10 @@ HASH=$(nix hash path $outPath) clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile +nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile --out-link $TEST_ROOT/result [[ -e $TEST_ROOT/profile ]] +[[ -e $TEST_ROOT/result ]] if ls $cacheDir/nar/*.zst &> /dev/null; then echo "files do exist" From e9b5704d1c3bdead2d9d259b3c443a391306b6d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2024 16:49:35 +0200 Subject: [PATCH 010/243] Add release note --- doc/manual/rl-next/nix-copy-flags.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/manual/rl-next/nix-copy-flags.md diff --git a/doc/manual/rl-next/nix-copy-flags.md b/doc/manual/rl-next/nix-copy-flags.md new file mode 100644 index 000000000..f5b2b9716 --- /dev/null +++ b/doc/manual/rl-next/nix-copy-flags.md @@ -0,0 +1,18 @@ +--- +synopsis: "`nix copy` supports `--profile` and `--out-link`" +prs: [11657] +--- + +The `nix copy` command now has flags `--profile` and `--out-link`, similar to `nix build`. `--profile` makes a profile point to the +top-level store path, while `--out-link` create symlinks to the top-level store paths. + +For example, when updating the local NixOS system profile from a NixOS system closure on a remote machine, instead of +``` +# nix copy --from ssh://server $path +# nix build --profile /nix/var/nix/profiles/system $path +``` +you can now do +``` +# nix copy --from ssh://server --profile /nix/var/nix/profiles/system $path +``` +The advantage is that this avoids a time window where *path* is not a garbage collector root, and so could be deleted by a concurrent `nix store gc` process. From 27ea43781371cad717077ae723b11a79c0d0fc78 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2024 16:54:11 +0200 Subject: [PATCH 011/243] Support fine-grained database schema migrations Backward-compatible schema changes (e.g. those that add tables or nullable columns) now no longer need a change to the global schema file (/nix/var/nix/db/schema). Thus, old Nix versions can continue to access the database. This is especially useful for schema changes required by experimental features. In particular, it replaces the ad-hoc handling of the schema changes for CA derivations (i.e. the file /nix/var/nix/db/ca-schema). Schema versions 8 and 10 could have been handled by this mechanism in a backward-compatible way as well. --- src/libstore/local-store.cc | 105 ++++++++++++++++++------------------ src/libstore/local-store.hh | 2 + 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eafdac0cd..f708bd1b0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -95,51 +95,6 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -static int getSchema(Path schemaPath) -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - auto s = readFile(schemaPath); - auto n = string2Int(s); - if (!n) - throw Error("'%1%' is corrupt", schemaPath); - curSchema = *n; - } - return curSchema; -} - -void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) -{ - const int nixCASchemaVersion = 4; - int curCASchema = getSchema(schemaPath); - if (curCASchema != nixCASchemaVersion) { - if (curCASchema > nixCASchemaVersion) { - throw Error("current Nix store ca-schema is version %1%, but I only support %2%", - curCASchema, nixCASchemaVersion); - } - - if (!lockFile(lockFd.get(), ltWrite, false)) { - printInfo("waiting for exclusive access to the Nix store for ca drvs..."); - lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks - lockFile(lockFd.get(), ltWrite, true); - } - - if (curCASchema == 0) { - static const char schema[] = - #include "ca-specific-schema.sql.gen.hh" - ; - db.exec(schema); - curCASchema = nixCASchemaVersion; - } - - if (curCASchema < 4) - throw Error("experimental CA schema version %d is no longer supported", curCASchema); - - writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); - lockFile(lockFd.get(), ltRead, true); - } -} - LocalStore::LocalStore( std::string_view scheme, PathView path, @@ -316,6 +271,10 @@ LocalStore::LocalStore( openDB(*state, false); + /* Legacy database schema migrations. Don't bump 'schema' for + new migrations; instead, add a migration to + upgradeDBSchema(). */ + if (curSchema < 8) { SQLiteTxn txn(state->db); state->db.exec("alter table ValidPaths add column ultimate integer"); @@ -342,13 +301,7 @@ LocalStore::LocalStore( else openDB(*state, false); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - if (!readOnly) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); - } else { - throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); - } - } + upgradeDBSchema(*state); /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, @@ -483,7 +436,17 @@ std::string LocalStore::getUri() int LocalStore::getSchema() -{ return nix::getSchema(schemaPath); } +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + auto s = readFile(schemaPath); + auto n = string2Int(s); + if (!n) + throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; + } + return curSchema; +} void LocalStore::openDB(State & state, bool create) { @@ -566,6 +529,42 @@ void LocalStore::openDB(State & state, bool create) } +void LocalStore::upgradeDBSchema(State & state) +{ + state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); + + std::set schemaMigrations; + + { + SQLiteStmt querySchemaMigrations; + querySchemaMigrations.create(state.db, "select migration from SchemaMigrations;"); + auto useQuerySchemaMigrations(querySchemaMigrations.use()); + while (useQuerySchemaMigrations.next()) + schemaMigrations.insert(useQuerySchemaMigrations.getStr(0)); + } + + auto doUpgrade = [&](const std::string & migrationName, const std::string & stmt) + { + if (schemaMigrations.contains(migrationName)) + return; + + debug("executing Nix database schema migration '%s'...", migrationName); + + SQLiteTxn txn(state.db); + state.db.exec(stmt + fmt(";\ninsert into SchemaMigrations values('%s')", migrationName)); + txn.commit(); + + schemaMigrations.insert(migrationName); + }; + + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + doUpgrade( + "20220326-ca-derivations", + #include "ca-specific-schema.sql.gen.hh" + ); +} + + /* To improve purity, users may want to make the Nix store a read-only bind mount. So make the Nix store writable for this process. */ void LocalStore::makeStoreWritable() diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 21848cc4d..83154d651 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -356,6 +356,8 @@ private: void openDB(State & state, bool create); + void upgradeDBSchema(State & state); + void makeStoreWritable(); uint64_t queryValidPathId(State & state, const StorePath & path); From 59246349d5b227cb3e35fbaef6b5468fbd2d3ec7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 8 Nov 2024 20:13:07 +0100 Subject: [PATCH 012/243] Make nix log command easier to copy --- src/libstore/build/derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 34ed16a38..3c0445fb6 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -991,7 +991,8 @@ Goal::Co DerivationGoal::buildDone() auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) ? "nix log" : "nix-store -l"; - msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + // Don't put quotes around the command. This is copy-pasted a ton, so don't make that error prone. + msg += fmt("For full logs, run " ANSI_BOLD "%s %s" ANSI_NORMAL, nixLogCommand, worker.store.printStorePath(drvPath)); } From 4b44fa0f06d74caf7664a23a771774d093598824 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 8 Nov 2024 20:17:13 +0100 Subject: [PATCH 013/243] Make nix log command easy to copy on its own line --- src/libstore/build/derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3c0445fb6..794be1568 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -991,8 +991,10 @@ Goal::Co DerivationGoal::buildDone() auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) ? "nix log" : "nix-store -l"; - // Don't put quotes around the command. This is copy-pasted a ton, so don't make that error prone. - msg += fmt("For full logs, run " ANSI_BOLD "%s %s" ANSI_NORMAL, + // The command is on a separate line for easy copying, such as with triple click. + // This message will be indented elsewhere, so removing the indentation before the + // command will not put it at the start of the line unfortunately. + msg += fmt("For full logs, run:\n " ANSI_BOLD "%s %s" ANSI_NORMAL, nixLogCommand, worker.store.printStorePath(drvPath)); } From e194e27f8574c193b1979cad8beef9ba5d7d584d Mon Sep 17 00:00:00 2001 From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com> Date: Thu, 25 Jan 2024 21:56:09 +0000 Subject: [PATCH 014/243] docker: Allow building for non-root user Add options uid, gid, uname, and gname to docker.nix. Setting these to e.g. 1000, 1000, "user", "user" will build an image which runs and allows using Nix as that user. --- docker.nix | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/docker.nix b/docker.nix index bd16b71cd..19479e57c 100644 --- a/docker.nix +++ b/docker.nix @@ -9,6 +9,10 @@ , maxLayers ? 100 , nixConf ? {} , flake-registry ? null +, uid ? 0 +, gid ? 0 +, uname ? "root" +, gname ? "root" }: let defaultPkgs = with pkgs; [ @@ -50,6 +54,15 @@ let description = "Unprivileged account (don't use!)"; }; + } // lib.optionalAttrs (uid != 0) { + "${uname}" = { + uid = uid; + shell = "${pkgs.bashInteractive}/bin/bash"; + home = "/home/${uname}"; + gid = gid; + groups = [ "${gname}" ]; + description = "Nix user"; + }; } // lib.listToAttrs ( map ( @@ -70,6 +83,8 @@ let root.gid = 0; nixbld.gid = 30000; nobody.gid = 65534; + } // lib.optionalAttrs (gid != 0) { + "${gname}".gid = gid; }; userToPasswd = ( @@ -150,6 +165,8 @@ let in "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; + userHome = if uid == 0 then "/root" else "/home/${uname}"; + baseSystem = let nixpkgs = pkgs.path; @@ -237,26 +254,26 @@ let mkdir -p $out/etc/nix cat $nixConfContentsPath > $out/etc/nix/nix.conf - mkdir -p $out/root - mkdir -p $out/nix/var/nix/profiles/per-user/root + mkdir -p $out${userHome} + mkdir -p $out/nix/var/nix/profiles/per-user/${uname} ln -s ${profile} $out/nix/var/nix/profiles/default-1-link ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default - ln -s /nix/var/nix/profiles/default $out/root/.nix-profile + ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile - ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link + ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels - mkdir -p $out/root/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels - echo "${channelURL} ${channelName}" > $out/root/.nix-channels + mkdir -p $out${userHome}/.nix-defexpr + ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels mkdir -p $out/bin $out/usr/bin ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh '' + (lib.optionalString (flake-registry-path != null) '' - nixCacheDir="/root/.cache/nix" + nixCacheDir="${userHome}/.cache/nix" mkdir -p $out$nixCacheDir globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" ln -s ${flake-registry-path} $out$globalFlakeRegistryPath @@ -268,7 +285,7 @@ let in pkgs.dockerTools.buildLayeredImageWithNixDb { - inherit name tag maxLayers; + inherit name tag maxLayers uid gid uname gname; contents = [ baseSystem ]; @@ -279,25 +296,28 @@ pkgs.dockerTools.buildLayeredImageWithNixDb { fakeRootCommands = '' chmod 1777 tmp chmod 1777 var/tmp + chown -R ${toString uid}:${toString gid} .${userHome} + chown -R ${toString uid}:${toString gid} nix ''; config = { - Cmd = [ "/root/.nix-profile/bin/bash" ]; + Cmd = [ "${userHome}/.nix-profile/bin/bash" ]; + User = "${toString uid}:${toString gid}"; Env = [ - "USER=root" + "USER=${uname}" "PATH=${lib.concatStringsSep ":" [ - "/root/.nix-profile/bin" + "${userHome}/.nix-profile/bin" "/nix/var/nix/profiles/default/bin" "/nix/var/nix/profiles/default/sbin" ]}" "MANPATH=${lib.concatStringsSep ":" [ - "/root/.nix-profile/share/man" + "${userHome}/.nix-profile/share/man" "/nix/var/nix/profiles/default/share/man" ]}" "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" - "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + "NIX_PATH=/nix/var/nix/profiles/per-user/${uname}/channels:${userHome}/.nix-defexpr/channels" ]; }; From 1cfb226b7269b14c34b2ef42e4c501e1ba851bb4 Mon Sep 17 00:00:00 2001 From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com> Date: Sun, 10 Nov 2024 21:31:02 +0000 Subject: [PATCH 015/243] tests/nixos: add nix-docker test --- tests/nixos/default.nix | 2 ++ tests/nixos/nix-docker.nix | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/nixos/nix-docker.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 17bfdea38..c5f4a23aa 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -124,6 +124,8 @@ in nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + nix-docker = runNixOSTestFor "x86_64-linux" ./nix-docker.nix; + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix new file mode 100644 index 000000000..5c21dfff6 --- /dev/null +++ b/tests/nixos/nix-docker.nix @@ -0,0 +1,39 @@ +# Test the container built by ../../docker.nix. + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + nixImage = import ../../docker.nix { + inherit (config.nodes.machine.nixpkgs) pkgs; + }; + nixUserImage = import ../../docker.nix { + inherit (config.nodes.machine.nixpkgs) pkgs; + name = "nix-user"; + uid = 1000; + gid = 1000; + uname = "user"; + gname = "user"; + }; + +in { + name = "nix-docker"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.diskSize = 4096; + }; + + testScript = { nodes }: '' + machine.succeed("mkdir -p /etc/containers") + machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""") + + machine.succeed("${pkgs.podman}/bin/podman load -i ${nixImage}") + machine.succeed("${pkgs.podman}/bin/podman run --rm nix nix --version") + + machine.succeed("${pkgs.podman}/bin/podman load -i ${nixUserImage}") + machine.succeed("${pkgs.podman}/bin/podman run --rm nix-user nix --version") + machine.succeed("[[ $(${pkgs.podman}/bin/podman run --rm nix-user stat -c %u /nix/store) = 1000 ]]") + ''; +} From 1dda18ef0a3c6d109b6e9fc2e1c7f93c9c1a4471 Mon Sep 17 00:00:00 2001 From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com> Date: Sun, 10 Nov 2024 21:31:32 +0000 Subject: [PATCH 016/243] doc/manual: add documentation for non-root container images --- .../source/installation/installing-docker.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual/source/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md index 6f77d6a57..9354c1a72 100644 --- a/doc/manual/source/installation/installing-docker.md +++ b/doc/manual/source/installation/installing-docker.md @@ -57,3 +57,21 @@ $ nix build ./\#hydraJobs.dockerImage.x86_64-linux $ docker load -i ./result/image.tar.gz $ docker run -ti nix:2.5pre20211105 ``` + +# Docker image with non-root Nix + +If you would like to run Nix in a container under a user other than `root`, +you can build an image with a non-root single-user installation of Nix +by specifying the `uid`, `gid`, `uname`, and `gname` arguments to `docker.nix`: + +```console +$ nix build --file docker.nix \ + --arg uid 1000 \ + --arg gid 1000 \ + --argstr uname user \ + --argstr gname user \ + --argstr name nix-user \ + --out-link nix-user.tar.gz +$ docker load -i nix-user.tar.gz +$ docker run -ti nix-user +``` From 11d3b017cfdce506bf46edf7f11ba923a67adee9 Mon Sep 17 00:00:00 2001 From: WxNzEMof <143541718+WxNzEMof@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:20:31 +0000 Subject: [PATCH 017/243] tests/nixos: add more thorough nix-docker tests --- tests/nixos/nix-docker-test.sh | 47 ++++++++++++++++++++++++++++++++++ tests/nixos/nix-docker.nix | 20 ++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/nixos/nix-docker-test.sh diff --git a/tests/nixos/nix-docker-test.sh b/tests/nixos/nix-docker-test.sh new file mode 100644 index 000000000..1f65e1a94 --- /dev/null +++ b/tests/nixos/nix-docker-test.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# docker.nix test script. Runs inside a built docker.nix container. + +set -eEuo pipefail + +export NIX_CONFIG='substituters = http://cache:5000?trusted=1' + +cd /tmp + +# Test getting a fetched derivation +test "$("$(nix-build -E '(import {}).hello')"/bin/hello)" = "Hello, world!" + +# Test building a simple derivation +# shellcheck disable=SC2016 +nix-build -E ' +let + pkgs = import {}; +in +builtins.derivation { + name = "test"; + system = builtins.currentSystem; + builder = "${pkgs.bash}/bin/bash"; + args = ["-c" "echo OK > $out"]; +}' +test "$(cat result)" = OK + +# Ensure #!/bin/sh shebang works +echo '#!/bin/sh' > ./shebang-test +echo 'echo OK' >> ./shebang-test +chmod +x ./shebang-test +test "$(./shebang-test)" = OK + +# Ensure #!/usr/bin/env shebang works +echo '#!/usr/bin/env bash' > ./shebang-test +echo 'echo OK' >> ./shebang-test +chmod +x ./shebang-test +test "$(./shebang-test)" = OK + +# Test nix-shell +{ + echo '#!/usr/bin/env nix-shell' + echo '#! nix-shell -i bash' + echo '#! nix-shell -p hello' + echo 'hello' +} > ./nix-shell-test +chmod +x ./nix-shell-test +test "$(./nix-shell-test)" = "Hello, world!" diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix index 5c21dfff6..dfd508988 100644 --- a/tests/nixos/nix-docker.nix +++ b/tests/nixos/nix-docker.nix @@ -17,23 +17,37 @@ let gname = "user"; }; + containerTestScript = ./nix-docker-test.sh; + in { name = "nix-docker"; - nodes.machine = - { config, lib, pkgs, ... }: - { virtualisation.diskSize = 4096; + nodes = + { machine = + { config, lib, pkgs, ... }: + { virtualisation.diskSize = 4096; + }; + cache = + { config, lib, pkgs, ... }: + { virtualisation.additionalPaths = [ pkgs.stdenv pkgs.hello ]; + services.harmonia.enable = true; + networking.firewall.allowedTCPPorts = [ 5000 ]; + }; }; testScript = { nodes }: '' + cache.wait_for_unit("harmonia.service") + machine.succeed("mkdir -p /etc/containers") machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""") machine.succeed("${pkgs.podman}/bin/podman load -i ${nixImage}") machine.succeed("${pkgs.podman}/bin/podman run --rm nix nix --version") + machine.succeed("${pkgs.podman}/bin/podman run --rm -i nix < ${containerTestScript}") machine.succeed("${pkgs.podman}/bin/podman load -i ${nixUserImage}") machine.succeed("${pkgs.podman}/bin/podman run --rm nix-user nix --version") + machine.succeed("${pkgs.podman}/bin/podman run --rm -i nix-user < ${containerTestScript}") machine.succeed("[[ $(${pkgs.podman}/bin/podman run --rm nix-user stat -c %u /nix/store) = 1000 ]]") ''; } From a2e4a4c2384789d92b300959995a7d9d0e9d725f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Nov 2024 19:26:39 +0100 Subject: [PATCH 018/243] callFunction: Use std::span This is a bit safer than having a separate nrArgs argument. --- src/libexpr-c/nix_api_expr.cc | 2 +- src/libexpr/eval.cc | 32 ++++++++++++++------------------ src/libexpr/eval.hh | 5 ++--- src/libexpr/primops.cc | 8 ++++---- src/libflake/flake/flake.cc | 2 +- 5 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 333e99460..6144a7986 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -67,7 +67,7 @@ nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_val if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.callFunction(fn->value, {(nix::Value * *) args, nargs}, value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e21f70553..6e82af1d8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -588,14 +588,14 @@ std::optional EvalState::getDoc(Value & v) if (isFunctor(v)) { try { Value & functor = *v.attrs()->find(sFunctor)->value; - Value * vp = &v; + Value * vp[] = {&v}; Value partiallyApplied; // The first paramater is not user-provided, and may be // handled by code that is opaque to the user, like lib.const = x: y: y; // So preferably we show docs that are relevant to the // "partially applied" function returned by e.g. `const`. // We apply the first argument: - callFunction(functor, 1, &vp, partiallyApplied, noPos); + callFunction(functor, vp, partiallyApplied, noPos); auto _level = addCallDepth(noPos); return getDoc(partiallyApplied); } @@ -1460,7 +1460,7 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } -void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) +void EvalState::callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos) { auto _level = addCallDepth(pos); @@ -1475,7 +1475,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto makeAppChain = [&]() { vRes = vCur; - for (size_t i = 0; i < nrArgs; ++i) { + for (size_t i = 0; i < args.size(); ++i) { auto fun2 = allocValue(); *fun2 = vRes; vRes.mkPrimOpApp(fun2, args[i]); @@ -1484,7 +1484,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & const Attr * functor; - while (nrArgs > 0) { + while (args.size() > 0) { if (vCur.isLambda()) { @@ -1587,15 +1587,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & throw; } - nrArgs--; - args += 1; + args = args.subspan(1); } else if (vCur.isPrimOp()) { size_t argsLeft = vCur.primOp()->arity; - if (nrArgs < argsLeft) { + if (args.size() < argsLeft) { /* We don't have enough arguments, so create a tPrimOpApp chain. */ makeAppChain(); return; @@ -1607,15 +1606,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (countCalls) primOpCalls[fn->name]++; try { - fn->fun(*this, vCur.determinePos(noPos), args, vCur); + fn->fun(*this, vCur.determinePos(noPos), args.data(), vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } - nrArgs -= argsLeft; - args += argsLeft; + args = args.subspan(argsLeft); } } @@ -1631,7 +1629,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto arity = primOp->primOp()->arity; auto argsLeft = arity - argsDone; - if (nrArgs < argsLeft) { + if (args.size() < argsLeft) { /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ makeAppChain(); return; @@ -1663,8 +1661,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & throw; } - nrArgs -= argsLeft; - args += argsLeft; + args = args.subspan(argsLeft); } } @@ -1675,13 +1672,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; try { - callFunction(*functor->value, 2, args2, vCur, functor->pos); + callFunction(*functor->value, args2, vCur, functor->pos); } catch (Error & e) { e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)"); throw; } - nrArgs--; - args++; + args = args.subspan(1); } else @@ -1724,7 +1720,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs.data(), v, pos); + state.callFunction(vFun, vArgs, v, pos); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7fe70af31..b0c79ab86 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -690,13 +690,12 @@ public: bool isFunctor(Value & fun); - // FIXME: use std::span - void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos); + void callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos); void callFunction(Value & fun, Value & arg, Value & vRes, const PosIdx pos) { Value * args[] = {&arg}; - callFunction(fun, 1, args, vRes, pos); + callFunction(fun, args, vRes, pos); } /** diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 45d9f86ac..aea623435 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -724,7 +724,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a /* Call the `operator' function with `e' as argument. */ Value newElements; - state.callFunction(*op->value, 1, &e, newElements, noPos); + state.callFunction(*op->value, {&e, 1}, newElements, noPos); state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure"); /* Add the values returned by the operator to the work set. */ @@ -2450,7 +2450,7 @@ bool EvalState::callPathFilter( // assert that type is not "unknown" Value * args []{&arg1, fileTypeToString(*this, st.type)}; Value res; - callFunction(*filterFun, 2, args, res, pos); + callFunction(*filterFun, args, res, pos); return forceBool(res, pos, "while evaluating the return value of the path filter function"); } @@ -3487,7 +3487,7 @@ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args for (auto [n, elem] : enumerate(args[2]->listItems())) { Value * vs []{vCur, elem}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(*args[0], 2, vs, *vCur, pos); + state.callFunction(*args[0], vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -3637,7 +3637,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value Value * vs[] = {a, b}; Value vBool; - state.callFunction(*args[0], 2, vs, vBool, noPos); + state.callFunction(*args[0], vs, vBool, noPos); return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort"); }; diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index edb76f861..19b622a34 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -816,7 +816,7 @@ void callFlake(EvalState & state, assert(vFetchFinalTree); Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree}; - state.callFunction(*vCallFlake, 3, args, vRes, noPos); + state.callFunction(*vCallFlake, args, vRes, noPos); } void initLib(const Settings & settings) From d4cec7511db159523a0101f36033393533bb4309 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Nov 2024 19:49:39 +0100 Subject: [PATCH 019/243] nix-everything: Remove libs, add dev and devdoc package outputs The new package output attributes are somewhat experimental, and provided for compatibility most of all. We'll see how well this goes before the changes proposed in https://github.com/NixOS/nix/issues/6507 --- packaging/everything.nix | 139 +++++++++++++++++++++++++++++---------- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 6f6bbc6aa..458d26c30 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -5,12 +5,10 @@ nix-util, nix-util-c, - nix-util-test-support, nix-util-tests, nix-store, nix-store-c, - nix-store-test-support, nix-store-tests, nix-fetchers, @@ -18,7 +16,6 @@ nix-expr, nix-expr-c, - nix-expr-test-support, nix-expr-tests, nix-flake, @@ -38,45 +35,84 @@ nix-external-api-docs, nix-perl-bindings, + + testers, + runCommand, }: +let + dev = stdenv.mkDerivation (finalAttrs: { + name = "nix-${nix-cli.version}-dev"; + pname = "nix"; + version = nix-cli.version; + dontUnpack = true; + dontBuild = true; + libs = map lib.getDev [ + nix-cmd + nix-expr + nix-expr-c + nix-fetchers + nix-flake + nix-main + nix-main-c + nix-store + nix-store-c + nix-util + nix-util-c + nix-perl-bindings + ]; + installPhase = '' + mkdir -p $out/nix-support + echo $libs >> $out/nix-support/propagated-build-inputs + ''; + passthru = { + tests = { + pkg-config = + testers.hasPkgConfigModules { + package = finalAttrs.finalPackage; + }; + }; + + # If we were to fully emulate output selection here, we'd confuse the Nix CLIs, + # because they rely on `drvPath`. + dev = finalAttrs.finalPackage.out; + + libs = throw "`nix.dev.libs` is not meant to be used; use `nix.libs` instead."; + }; + meta = { + pkgConfigModules = [ + "nix-cmd" + "nix-expr" + "nix-expr-c" + "nix-fetchers" + "nix-flake" + "nix-main" + "nix-main-c" + "nix-store" + "nix-store-c" + "nix-util" + "nix-util-c" + ]; + }; + }); + devdoc = buildEnv { + name = "nix-${nix-cli.version}-devdoc"; + paths = [ + nix-internal-api-docs + nix-external-api-docs + ]; + }; + +in (buildEnv { name = "nix-${nix-cli.version}"; paths = [ - nix-util - nix-util-c - nix-util-test-support - nix-util-tests - - nix-store - nix-store-c - nix-store-test-support - nix-store-tests - - nix-fetchers - nix-fetchers-tests - - nix-expr - nix-expr-c - nix-expr-test-support - nix-expr-tests - - nix-flake - nix-flake-tests - - nix-main - nix-main-c - - nix-cmd + # unfortunately, `buildEnv` doesn't support multiple outputs nix-cli + # TODO: separate doc output attribute? nix-manual - nix-internal-api-docs - nix-external-api-docs - - ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ - nix-perl-bindings ]; meta.mainProgram = "nix"; @@ -85,12 +121,25 @@ doInstallCheck = true; checkInputs = [ - # Actually run the unit tests too + # Make sure the unit tests have passed nix-util-tests.tests.run nix-store-tests.tests.run nix-expr-tests.tests.run + nix-fetchers-tests.tests.run nix-flake-tests.tests.run - ]; + + # dev bundle is ok + # (checkInputs must be empty paths??) + (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out") + ] ++ + (if stdenv.buildPlatform.canExecute stdenv.hostPlatform + then [ + # TODO: add perl.tests + nix-perl-bindings + ] + else [ + nix-perl-bindings + ]); installCheckInputs = [ nix-functional-tests ]; @@ -128,5 +177,25 @@ nix-main-c ; }; + + tests = prevAttrs.passthru.tests or {} // { + # TODO: create a proper fixpoint and: + # pkg-config = + # testers.hasPkgConfigModules { + # package = finalPackage; + # }; + }; + + /** + A derivation referencing the `dev` outputs of the Nix libraries. + */ + inherit dev; + inherit devdoc; + outputs = [ "out" "dev" "devdoc" ]; + all = lib.attrValues (lib.genAttrs finalAttrs.passthru.outputs (outName: finalAttrs.finalPackage.${outName})); + }; + meta = prevAttrs.meta // { + description = "The Nix package manager"; + pkgConfigModules = dev.meta.pkgConfigModules; }; }) From f312a7cfffbe2959fd2698489284ff8081834a5c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Nov 2024 20:05:29 +0100 Subject: [PATCH 020/243] nix-everything: Make doc a separate package output --- packaging/everything.nix | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 458d26c30..b09b9d2a9 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -107,12 +107,8 @@ in (buildEnv { name = "nix-${nix-cli.version}"; paths = [ - # unfortunately, `buildEnv` doesn't support multiple outputs - nix-cli - - # TODO: separate doc output attribute? - nix-manual + nix-manual.man ]; meta.mainProgram = "nix"; @@ -191,7 +187,8 @@ in */ inherit dev; inherit devdoc; - outputs = [ "out" "dev" "devdoc" ]; + doc = nix-manual; + outputs = [ "out" "dev" "devdoc" "doc" ]; all = lib.attrValues (lib.genAttrs finalAttrs.passthru.outputs (outName: finalAttrs.finalPackage.${outName})); }; meta = prevAttrs.meta // { From b9f8c4af4057c2ed0ec5d1ff16ac49e0612ad57c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Thu, 14 Nov 2024 02:06:59 +0300 Subject: [PATCH 021/243] fix(libmain/common-args): do not exceed maximum allowed verbosity This patch gets rid of UB when verbosity exceeds the maximum logging value of `lvlVomit = 7` and reaches invalid values (e.g. 8). This is actually triggered in functional tests. There are too many occurrences to list, but here's one from the UBSAN log: ../src/libstore/gc.cc:610:5: runtime error: load of value 8, which is not a valid value for type 'Verbosity' --- src/libmain/common-args.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 768b2177c..13d358623 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -17,7 +17,9 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .shortName = 'v', .description = "Increase the logging verbosity level.", .category = loggingCategory, - .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, + .handler = {[]() { + verbosity = (Verbosity) std::min>(verbosity + 1, lvlVomit); + }}, }); addFlag({ From e53e0a04f4be61ac085d87480fb89c96f7ae9261 Mon Sep 17 00:00:00 2001 From: dbdr Date: Thu, 14 Nov 2024 09:16:00 +0100 Subject: [PATCH 022/243] Fix typo in nix-collect-garbage.md --- doc/manual/source/command-ref/nix-collect-garbage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 2136d28e9..bd05f2816 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -36,7 +36,7 @@ Instead, it looks in a few locations, and acts on all profiles it finds there: > > Not stable; subject to change > - > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > Do not rely on this functionality; it just exists for migration purposes and may change in the future. > These deprecated paths remain a private implementation detail of Nix. `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. From 33a0fa882f868102f3fca8f0d7547f3727be1244 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Nov 2024 15:40:33 +0100 Subject: [PATCH 023/243] nix path-info: Don't write to std::cout directly This interferes with the progress bar, resulting in output like evaluating derivation 'git+file:///home/eelco/Dev/nix-master#packages.x86_64-linux.default'/nix/store/zz8v96j5md952x0mxfix12xqnvq5qv5x-nix-2.26.0pre20241114_a95f6ea.drv --- src/nix/path-info.cc | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index e7cfb6e7a..8e3d0406d 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -139,12 +139,12 @@ struct CmdPathInfo : StorePathsCommand, MixJSON Category category() override { return catSecondary; } - void printSize(uint64_t value) + void printSize(std::ostream & str, uint64_t value) { if (humanReadable) - std::cout << fmt("\t%s", renderSize(value, true)); + str << fmt("\t%s", renderSize(value, true)); else - std::cout << fmt("\t%11d", value); + str << fmt("\t%11d", value); } void run(ref store, StorePaths && storePaths) override @@ -154,11 +154,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << pathInfoToJSON( + logger->cout(pathInfoToJSON( *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - showClosureSize).dump(); + showClosureSize).dump()); } else { @@ -167,30 +167,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON auto info = store->queryPathInfo(storePath); auto storePathS = store->printStorePath(info->path); - std::cout << storePathS; + std::ostringstream str; + + str << storePathS; if (showSize || showClosureSize || showSigs) - std::cout << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' '); + str << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' '); if (showSize) - printSize(info->narSize); + printSize(str, info->narSize); if (showClosureSize) { StorePathSet closure; store->computeFSClosure(storePath, closure, false, false); - printSize(getStoreObjectsTotalSize(*store, closure)); + printSize(str, getStoreObjectsTotalSize(*store, closure)); } if (showSigs) { - std::cout << '\t'; + str << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); - std::cout << concatStringsSep(" ", ss); + str << concatStringsSep(" ", ss); } - std::cout << std::endl; + logger->cout(str.str()); } } From 3e4a83f53b343a7fe2ba4faaf4e3932c99ee438b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Nov 2024 16:12:14 +0100 Subject: [PATCH 024/243] Use range-based for --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6e82af1d8..0283a14d6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1475,10 +1475,10 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, auto makeAppChain = [&]() { vRes = vCur; - for (size_t i = 0; i < args.size(); ++i) { + for (auto arg : args) { auto fun2 = allocValue(); *fun2 = vRes; - vRes.mkPrimOpApp(fun2, args[i]); + vRes.mkPrimOpApp(fun2, arg); } }; From 2f3764acbb96ab687e612fdd81bdf58ac5f0e605 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 15 Nov 2024 11:35:27 +0100 Subject: [PATCH 025/243] .github/ci: Add nix-docker test We still have room to spare in vm_tests, as it's quicker than `nix flake check` --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27f60574e..a3b7b06d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,7 +194,13 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + - run: | + nix build -L \ + .#hydraJobs.tests.functional_user \ + .#hydraJobs.tests.githubFlakes \ + .#hydraJobs.tests.nix-docker \ + .#hydraJobs.tests.tarballFlakes \ + ; flake_regressions: needs: vm_tests From c9433c0d1834362305c4f5ca2802fca97d999044 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 15 Nov 2024 11:37:17 +0100 Subject: [PATCH 026/243] .github/ci: Push docker only when test succeeds --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b7b06d4..cb97fd211 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,7 +128,7 @@ jobs: - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" docker_push_image: - needs: [check_secrets, tests] + needs: [check_secrets, tests, vm_tests] permissions: contents: read packages: write From 3f6855c31b8d35b39148a86c2c28c7e2f367b739 Mon Sep 17 00:00:00 2001 From: myclevorname <140354451+myclevorname@users.noreply.github.com> Date: Sun, 17 Nov 2024 14:12:27 -0500 Subject: [PATCH 027/243] doc/nix fmt: Mention nixfmt-rfc-style instead of nixfmt(-classic) --- src/nix/fmt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/fmt.md b/src/nix/fmt.md index a2afde61c..b4693eb65 100644 --- a/src/nix/fmt.md +++ b/src/nix/fmt.md @@ -22,13 +22,13 @@ With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): } ``` -With [nixfmt](https://github.com/serokell/nixfmt): +With [nixfmt](https://github.com/NixOS/nixfmt): ```nix # flake.nix { outputs = { nixpkgs, self }: { - formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt; + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style; }; } ``` From d65fac0fc461e5b62373b86401b845df6c698177 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 18 Nov 2024 15:08:32 +0100 Subject: [PATCH 028/243] Add --print-errorlogs to mesonCheckFlags This prints the error logs in the tests, including when they're run with `checkPhase` in the dev shell. --- packaging/dependencies.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 13766f2c0..a8005ce16 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -70,6 +70,9 @@ let pkgs.buildPackages.meson pkgs.buildPackages.ninja ] ++ prevAttrs.nativeBuildInputs or []; + mesonCheckFlags = prevAttrs.mesonCheckFlags or [] ++ [ + "--print-errorlogs" + ]; }; mesonBuildLayer = finalAttrs: prevAttrs: From 428af8c66f0918f5852080b06268498e5021bfe8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Nov 2024 16:28:12 +0100 Subject: [PATCH 029/243] tests/functional/flakes/develop.sh: Don't hang The bash shell started by `nix develop` waited forever for stdin input. Fixes #11827. --- tests/functional/flakes/develop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh index b3e438e99..c222f0fbb 100755 --- a/tests/functional/flakes/develop.sh +++ b/tests/functional/flakes/develop.sh @@ -122,7 +122,7 @@ expectStderr 1 nix develop --unset-env-var FOO --set-env-var FOO 'BAR' --no-writ grepQuiet "error: Cannot set environment variable 'FOO' that is unset with '--unset-env-var'" # Check that multiple `--ignore-env`'s are okay. -expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello +expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello < /dev/null # Determine the bashInteractive executable. nix build --no-write-lock-file './nixpkgs#bashInteractive' --out-link ./bash-interactive From c4b95dbdd1fb45bbc7ad1fc921ebdf81789e22b3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 18 Nov 2024 16:45:18 +0100 Subject: [PATCH 030/243] Fix issue 11892 It seems that I copied the expression for baseDir thoughtlessly and did not come back to it. - `baseDir` was only used in the `fromArgs` branch. - `fromArgs` is true when `packages` is true. --- src/nix-build/nix-build.cc | 10 ++++++---- tests/functional/nix-shell.sh | 29 +++++++++++++++++++++++++++++ tests/functional/shell.nix | 6 +++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c394836da..de01e1afc 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -340,13 +340,15 @@ static void main_nix_build(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : remainingArgs) { - auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i; - - if (fromArgs) + if (fromArgs) { + auto shebangBaseDir = absPath(dirOf(script)); exprs.push_back(state->parseExprFromString( std::move(i), - (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".") + (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) + ? lookupFileArg(*state, shebangBaseDir) + : state->rootPath(".") )); + } else { auto absolute = i; try { diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 2b78216f4..b054b7f75 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -167,6 +167,35 @@ EOF chmod a+x $TEST_ROOT/marco/polo/default.nix (cd $TEST_ROOT/marco && ./polo/default.nix | grepQuiet "Polo") +# https://github.com/NixOS/nix/issues/11892 +mkdir $TEST_ROOT/issue-11892 +cat >$TEST_ROOT/issue-11892/shebangscript <$TEST_ROOT/issue-11892/my_package.nix < \$out/bin/my_package + cat \$out/bin/my_package + chmod a+x \$out/bin/my_package + ''; +} +EOF +chmod a+x $TEST_ROOT/issue-11892/shebangscript +$TEST_ROOT/issue-11892/shebangscript \ + | tee /dev/stderr \ + | grepQuiet "ok baz11892" + ##################### # Flake equivalents # diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 9cae14b78..4b1a0623a 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -37,7 +37,7 @@ let pkgs = rec { mkdir -p $out ln -s ${setupSh} $out/setup ''; - }; + } // { inherit mkDerivation; }; shellDrv = mkDerivation { name = "shellDrv"; @@ -94,5 +94,9 @@ let pkgs = rec { chmod a+rx $out/bin/ruby ''; + inherit (cfg) shell; + + callPackage = f: args: f (pkgs // args); + inherit pkgs; }; in pkgs From e224a35a77b588aed2bc29bf36cc4274098e7a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Nov 2024 11:25:26 +0100 Subject: [PATCH 031/243] docs/flake: document how to build a pull request It's not so common knowledge that forges also expose pull requests as git refs. But it's actually a cool way of quickly testing someones contribution, so I found it worth specifically mentioning it. --- src/nix/flake.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index 2b999431c..8f0f9936c 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -84,6 +84,8 @@ Here are some examples of flake references in their URL-like representation: repository on GitHub. * `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the `nixpkgs` repository. +* `github:NixOS/nixpkgs/pull/357207/head`: The `357207` pull request + of the nixpkgs repository. * `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A specific revision of the `nixpkgs` repository. * `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory From dd4838974eb8afbde4cf3dc60ffd327084affb06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Nov 2024 13:24:11 +0100 Subject: [PATCH 032/243] document shallow clone options in git fetchers --- src/nix/flake.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index 8f0f9936c..5412afcdd 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -245,6 +245,9 @@ Currently the `type` attribute can be one of the following: * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` + * `git+https://example.org/my/repo?shallow=1` A shallow clone of the repository. + For large repositories, the shallow clone option can significantly speed up fresh clones compared + to non-shallow clones, while still providing faster updates than other fetch methods such as `tarball:` or `github:`. * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` * `git+file:///home/my-user/some-repo/some-repo` From 850281908cd65b7ccfdfe17b1e4a43f8ec59ef9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 16:50:13 +0100 Subject: [PATCH 033/243] Clean up flakeref parsing This factors out some commonality in calling fromURL() and handling the "dir" parameter into a fromParsedURL() helper function. --- src/libflake/flake/flakeref.cc | 43 ++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 01fe747f9..ed47ad737 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -67,6 +67,20 @@ std::optional maybeParseFlakeRef( } } +static std::pair fromParsedURL( + const fetchers::Settings & fetchSettings, + ParsedURL && parsedURL, + bool isFlake) +{ + auto dir = getOr(parsedURL.query, "dir", ""); + parsedURL.query.erase("dir"); + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment); +} + std::pair parsePathFlakeRefWithFragment( const fetchers::Settings & fetchSettings, const std::string & url, @@ -89,7 +103,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); } if (baseDir) { @@ -153,6 +167,7 @@ std::pair parsePathFlakeRefWithFragment( .authority = "", .path = flakeRoot, .query = query, + .fragment = fragment, }; if (subdir != "") { @@ -164,9 +179,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); + return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -185,11 +198,12 @@ std::pair parsePathFlakeRefWithFragment( attrs.insert_or_assign("path", path); return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); -}; +} - -/* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ +/** + * Check if `url` is a flake ID. This is an abbreviated syntax for + * `flake:?ref=&rev=`. + */ static std::optional> parseFlakeIdRef( const fetchers::Settings & fetchSettings, const std::string & url, @@ -227,22 +241,11 @@ std::optional> parseURLFlakeRef( bool isFlake ) { - ParsedURL parsedURL; try { - parsedURL = parseURL(url); + return fromParsedURL(fetchSettings, parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } - - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); } std::pair parseFlakeRefWithFragment( From 868b4d37ea8e59d76afbaef4c82315e23b5fa8f4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 16:59:38 +0100 Subject: [PATCH 034/243] nix flake init: Operate on a SourcePath Cherry-picked from lazy-trees. --- src/nix/flake.cc | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3a54763a1..ce2faacb0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -891,37 +891,32 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto cursor = installable.getCursor(*evalState); - auto templateDirAttr = cursor->getAttr("path"); - auto templateDir = templateDirAttr->getString(); - - if (!store->isInStore(templateDir)) - evalState->error( - "'%s' was not found in the Nix store\n" - "If you've set '%s' to a string, try using a path instead.", - templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); + auto templateDirAttr = cursor->getAttr("path")->forceValue(); + NixStringContext context; + auto templateDir = evalState->coerceToPath(noPos, templateDirAttr, context, ""); std::vector changedFiles; std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const fs::path & from, const fs::path & to) + std::function copyDir; + copyDir = [&](const SourcePath & from, const fs::path & to) { fs::create_directories(to); - for (auto & entry : fs::directory_iterator{from}) { + for (auto & [name, entry] : from.readDirectory()) { checkInterrupt(); - auto from2 = entry.path(); - auto to2 = to / entry.path().filename(); - auto st = entry.symlink_status(); + auto from2 = from / name; + auto to2 = to / name; + auto st = from2.lstat(); auto to_st = fs::symlink_status(to2); - if (fs::is_directory(st)) + if (st.type == SourceAccessor::tDirectory) copyDir(from2, to2); - else if (fs::is_regular_file(st)) { - auto contents = readFile(from2.string()); + else if (st.type == SourceAccessor::tRegular) { + auto contents = from2.readFile(); if (fs::exists(to_st)) { auto contents2 = readFile(to2.string()); if (contents != contents2) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); @@ -930,18 +925,18 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else writeFile(to2, contents); } - else if (fs::is_symlink(st)) { - auto target = fs::read_symlink(from2); + else if (st.type == SourceAccessor::tSymlink) { + auto target = from2.readLink(); if (fs::exists(to_st)) { if (fs::read_symlink(to2) != target) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); } continue; } else - fs::create_symlink(target, to2); + fs::create_symlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); @@ -957,14 +952,14 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); } - auto welcomeText = cursor->maybeGetAttr("welcomeText"); - if (welcomeText) { + + if (auto welcomeText = cursor->maybeGetAttr("welcomeText")) { notice("\n"); notice(renderMarkdownToTerminal(welcomeText->getString())); } if (!conflictedFiles.empty()) - throw Error("Encountered %d conflicts - see above", conflictedFiles.size()); + throw Error("encountered %d conflicts - see above", conflictedFiles.size()); } }; From f1b4f14055077e660b7aa5b4859ce6965b62b886 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 17:30:38 +0100 Subject: [PATCH 035/243] Trivial changes from lazy-trees --- src/libexpr/primops.cc | 7 ++++--- src/libfetchers/fetch-to-store.cc | 2 ++ src/libfetchers/meson.build | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aea623435..42136b467 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1603,7 +1603,8 @@ static RegisterPrimOp primop_placeholder({ *************************************************************/ -/* Convert the argument to a path. !!! obsolete? */ +/* Convert the argument to a path and then to a string (confusing, + eh?). !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -2614,13 +2615,13 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else state.error( - "unsupported argument '%1%' to 'addPath'", + "unsupported argument '%1%' to 'builtins.path'", state.symbols[attr.name] ).atPos(attr.pos).debugThrow(); } if (!path) state.error( - "missing required 'path' attribute in the first argument to builtins.path" + "missing required 'path' attribute in the first argument to 'builtins.path'" ).atPos(pos).debugThrow(); if (name.empty()) name = path->baseName(); diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 65aa72a6c..fe347a59d 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -44,6 +44,8 @@ StorePath fetchToStore( : store.addToStore( name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); + if (cacheKey && mode == FetchMode::Copy) fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d4f202796..ff638578d 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -52,15 +52,15 @@ sources = files( 'fetch-to-store.cc', 'fetchers.cc', 'filtering-source-accessor.cc', - 'git.cc', 'git-utils.cc', + 'git.cc', 'github.cc', 'indirect.cc', 'mercurial.cc', 'mounted-source-accessor.cc', 'path.cc', - 'store-path-accessor.cc', 'registry.cc', + 'store-path-accessor.cc', 'tarball.cc', ) @@ -71,10 +71,10 @@ headers = files( 'cache.hh', 'fetch-settings.hh', 'fetch-to-store.hh', + 'fetchers.hh', 'filtering-source-accessor.hh', 'git-utils.hh', 'mounted-source-accessor.hh', - 'fetchers.hh', 'registry.hh', 'store-path-accessor.hh', 'tarball.hh', From a58e38dab7f6a9b9d217f6163fe8b565f19a6405 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 19 Nov 2024 17:30:58 +0100 Subject: [PATCH 036/243] Make EvalState::getBuiltin safe for missing attr --- src/libexpr/eval.cc | 6 +++++- src/libexpr/eval.hh | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0283a14d6..dd14f485e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -525,7 +525,11 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value & EvalState::getBuiltin(const std::string & name) { - return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value; + auto it = baseEnv.values[0]->attrs()->get(symbols.create(name)); + if (it) + return *it->value; + else + throw EvalError("builtin '%1%' not found", name); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 742f1cafe..01c725930 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -623,6 +623,11 @@ private: public: + /** + * Retrieve a specific builtin, equivalent to evaluating `builtins.${name}`. + * @param name The attribute name of the builtin to retrieve. + * @throws EvalError if the builtin does not exist. + */ Value & getBuiltin(const std::string & name); struct Doc From af07f33d37410de8c97bb64f2d77214c1f4640b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 18:03:31 +0100 Subject: [PATCH 037/243] resolveLookupPathPath(): Return a SourcePath instead of a string Cherry-picked from lazy-trees. --- src/libcmd/common-eval-args.cc | 6 +++--- src/libexpr/eval-settings.hh | 8 +++----- src/libexpr/eval.cc | 23 +++++++++++------------ src/libexpr/eval.hh | 6 +++--- tests/functional/restricted.sh | 2 +- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index ccbf957d9..de967e3fe 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -29,13 +29,13 @@ EvalSettings evalSettings { { { "flake", - [](ref store, std::string_view rest) { + [](EvalState & state, std::string_view rest) { experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); - auto storePath = flakeRef.resolve(store).fetchTree(store).first; - return store->toRealPath(storePath); + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; + return state.rootPath(state.store->toRealPath(storePath)); }, }, }, diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 115e3ee50..a8fcce539 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -3,10 +3,11 @@ #include "config.hh" #include "ref.hh" +#include "source-path.hh" namespace nix { -class Store; +class EvalState; struct EvalSettings : Config { @@ -18,11 +19,8 @@ struct EvalSettings : Config * * The return value is (a) whether the entry was valid, and, if so, * what does it map to. - * - * @todo Return (`std::optional` of) `SourceAccssor` or something - * more structured instead of mere `std::string`? */ - using LookupPathHook = std::optional(ref store, std::string_view); + using LookupPathHook = std::optional(EvalState & state, std::string_view); /** * Map from "scheme" to a `LookupPathHook`. diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0283a14d6..83a1cf4e9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3025,8 +3025,8 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ if (!rOpt) continue; auto r = *rOpt; - Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); + auto res = (r / CanonPath(suffix)).resolveSymlinks(); + if (res.pathExists()) return res; } if (hasPrefix(path, "nix/")) @@ -3041,13 +3041,13 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; auto i = lookupPathResolved.find(value); if (i != lookupPathResolved.end()) return i->second; - auto finish = [&](std::string res) { + auto finish = [&](SourcePath res) { debug("resolved search path element '%s' to '%s'", value, res); lookupPathResolved.emplace(value, res); return res; @@ -3060,7 +3060,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa fetchSettings, EvalSettings::resolvePseudoUrl(value)); auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); - return finish(store->toRealPath(storePath)); + return finish(rootPath(store->toRealPath(storePath))); } catch (Error & e) { logWarning({ .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) @@ -3072,29 +3072,29 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa auto scheme = value.substr(0, colPos); auto rest = value.substr(colPos + 1); if (auto * hook = get(settings.lookupPathHooks, scheme)) { - auto res = (*hook)(store, rest); + auto res = (*hook)(*this, rest); if (res) return finish(std::move(*res)); } } { - auto path = absPath(value); + auto path = rootPath(value); /* Allow access to paths in the search path. */ if (initAccessControl) { - allowPath(path); - if (store->isInStore(path)) { + allowPath(path.path.abs()); + if (store->isInStore(path.path.abs())) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(path).first, closure); + store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); for (auto & p : closure) allowPath(p); } catch (InvalidPath &) { } } } - if (pathExists(path)) + if (path.pathExists()) return finish(std::move(path)); else { logWarning({ @@ -3105,7 +3105,6 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa debug("failed to resolve search path element '%s'", value); return std::nullopt; - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 742f1cafe..6c0bae451 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -347,7 +347,7 @@ private: LookupPath lookupPath; - std::map> lookupPathResolved; + std::map> lookupPathResolved; /** * Cache used by prim_match(). @@ -452,9 +452,9 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt` + * If it is not found, return `std::nullopt`. */ - std::optional resolveLookupPathPath( + std::optional resolveLookupPathPath( const LookupPath::Path & elem, bool initAccessControl = false); diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 00ee4ddc8..a92a9b8a3 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -23,7 +23,7 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=./simple.nix -I src2=./conf (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "was not found in the Nix search path" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}") From 8a36d2d8a77dc3f3214ac6f7fd67cbe15ec6c706 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 19 Nov 2024 18:23:05 +0100 Subject: [PATCH 038/243] Add EvalState::getBuiltins --- src/libexpr-test-support/tests/libexpr.hh | 6 ++++++ src/libexpr-tests/eval.cc | 25 ++++++++++++++++++++++- src/libexpr/eval.cc | 8 +++++++- src/libexpr/eval.hh | 6 ++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/libexpr-test-support/tests/libexpr.hh b/src/libexpr-test-support/tests/libexpr.hh index 045607e87..095ea1d0e 100644 --- a/src/libexpr-test-support/tests/libexpr.hh +++ b/src/libexpr-test-support/tests/libexpr.hh @@ -40,6 +40,12 @@ namespace nix { return v; } + Value * maybeThunk(std::string input, bool forceValue = true) { + Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); + assert(e); + return e->maybeThunk(state, state.baseEnv); + } + Symbol createSymbol(const char * value) { return state.symbols.create(value); } diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 93d3f658f..61f6be0db 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -138,4 +138,27 @@ TEST(nix_isAllowedURI, non_scheme_colon) { ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); } -} // namespace nix \ No newline at end of file +class EvalStateTest : public LibExprTest {}; + +TEST_F(EvalStateTest, getBuiltins_ok) { + auto evaled = maybeThunk("builtins"); + auto & builtins = state.getBuiltins(); + ASSERT_TRUE(builtins.type() == nAttrs); + ASSERT_EQ(evaled, &builtins); +} + +TEST_F(EvalStateTest, getBuiltin_ok) { + auto & builtin = state.getBuiltin("toString"); + ASSERT_TRUE(builtin.type() == nFunction); + // FIXME + // auto evaled = maybeThunk("builtins.toString"); + // ASSERT_EQ(evaled, &builtin); + auto & builtin2 = state.getBuiltin("true"); + ASSERT_EQ(state.forceBool(builtin2, noPos, "in unit test"), true); +} + +TEST_F(EvalStateTest, getBuiltin_fail) { + ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError); +} + +} // namespace nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dd14f485e..b1b7a0fe6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -523,13 +523,19 @@ Value * EvalState::addPrimOp(PrimOp && primOp) } +Value & EvalState::getBuiltins() +{ + return *baseEnv.values[0]; +} + + Value & EvalState::getBuiltin(const std::string & name) { auto it = baseEnv.values[0]->attrs()->get(symbols.create(name)); if (it) return *it->value; else - throw EvalError("builtin '%1%' not found", name); + error("builtin '%1%' not found", name).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 01c725930..a14e88f0e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -630,6 +630,12 @@ public: */ Value & getBuiltin(const std::string & name); + /** + * Retrieve the `builtins` attrset, equivalent to evaluating the reference `builtins`. + * Always returns an attribute set value. + */ + Value & getBuiltins(); + struct Doc { Pos pos; From 5c258d7e255946c5d78ca8116966160e07455347 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 19 Nov 2024 18:32:09 +0100 Subject: [PATCH 039/243] refactor: Use EvalState::getBuiltins() --- src/libexpr/eval.cc | 6 +++--- src/libexpr/primops.cc | 2 +- src/nix/main.cc | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b1b7a0fe6..63c2e8a71 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -448,7 +448,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) /* Install value the base environment. */ staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v)); + getBuiltins().payload.attrs->push_back(Attr(symbols.create(name2), v)); } } @@ -516,7 +516,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) else { staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v)); + getBuiltins().payload.attrs->push_back(Attr(symbols.create(primOp.name), v)); } return v; @@ -531,7 +531,7 @@ Value & EvalState::getBuiltins() Value & EvalState::getBuiltin(const std::string & name) { - auto it = baseEnv.values[0]->attrs()->get(symbols.create(name)); + auto it = getBuiltins().attrs()->get(symbols.create(name)); if (it) return *it->value; else diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aea623435..50709b18a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4937,7 +4937,7 @@ void EvalState::createBaseEnv() /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ - baseEnv.values[0]->payload.attrs->sort(); + getBuiltins().payload.attrs->sort(); staticBaseEnv->sort(); diff --git a/src/nix/main.cc b/src/nix/main.cc index eff2d60a4..b0e26e093 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -435,7 +435,8 @@ void mainWrapped(int argc, char * * argv) evalSettings.pureEval = false; EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto builtinsJson = nlohmann::json::object(); - for (auto & builtin : *state.baseEnv.values[0]->attrs()) { + for (auto & builtinPtr : state.getBuiltins().attrs()->lexicographicOrder(state.symbols)) { + auto & builtin = *builtinPtr; auto b = nlohmann::json::object(); if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp(); From e948c8e03304a899893fffc1238d0d6025ed2564 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 19 Nov 2024 18:59:08 +0100 Subject: [PATCH 040/243] Bump fetcher cache version We're getting more reports in https://github.com/NixOS/nix/issues/10985 It appears that something hasn't gone right process-wise. I find this mistake not to be worth investigating, but rather something to pay attention to going forward. Let's nip this in the bud. Closes https://github.com/NixOS/nix/issues/10985 --- src/libfetchers/cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index b0b6cb887..6c2241f3a 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -36,7 +36,7 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite"; + auto dbPath = getCacheDir() + "/fetcher-cache-v3.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); From 4fca22b0dc6fe8054cb18e4e622b6dd2e3b8635c Mon Sep 17 00:00:00 2001 From: Gavin John Date: Tue, 19 Nov 2024 11:52:45 -0800 Subject: [PATCH 041/243] Update issue and pull request templates --- .github/ISSUE_TEMPLATE/bug_report.md | 39 +++++++++++++------ .github/ISSUE_TEMPLATE/feature_request.md | 35 ++++++++++++----- .github/ISSUE_TEMPLATE/installer.md | 17 ++++++-- .../ISSUE_TEMPLATE/missing_documentation.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 8 ++-- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 984f9a9ea..7ea82dcbc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,36 +1,51 @@ --- name: Bug report -about: Create a report to help us improve +about: Report unexpected or incorrect behaviour title: '' labels: bug assignees: '' --- -**Describe the bug** +## Describe the bug -A clear and concise description of what the bug is. + -**Steps To Reproduce** +## Steps To Reproduce 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** +## Expected behavior -A clear and concise description of what you expected to happen. + -**`nix-env --version` output** +## Metadata -**Additional context** + -Add any other context about the problem here. +## Additional context -**Priorities** + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 42c658b52..c75a46951 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,24 +1,39 @@ --- name: Feature request -about: Suggest an idea for this project +about: Suggest a new feature title: '' labels: feature assignees: '' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +## Is your feature request related to a problem? -**Describe the solution you'd like** -A clear and concise description of what you want to happen. + -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +## Proposed solution -**Additional context** -Add any other context or screenshots about the feature request here. + -**Priorities** +## Alternative solutions + + + +## Additional context + + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open feature issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 3768a49c9..ed5e1ce87 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -23,14 +23,25 @@ assignees: ''
Output -```log + - +```log ```
-## Priorities +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open installer issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source +[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer + +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index cf663e28d..6c334b722 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -26,6 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Priorities +--- Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 69da87db7..c6843d86f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,10 +17,12 @@ so you understand the process and the expectations. --> -# Motivation +## Motivation + -# Context +## Context + @@ -29,7 +31,7 @@ so you understand the process and the expectations. -# Priorities and Process +--- Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). From df9ccdf31b24439c0aac4c2fad6a04529aaf6f50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Nov 2024 17:35:32 +0100 Subject: [PATCH 042/243] BasicDerivation: Add applyRewrites() method This is the first part of rewriteDerivation() factored out into its own method. It's not used anywhere else at the moment, but it's useful on lazy-trees for rewriting virtual paths. --- src/libstore/derivations.cc | 26 ++++++++++++++------------ src/libstore/derivations.hh | 6 ++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9b6f67852..1f37b0c38 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1017,29 +1017,31 @@ std::string hashPlaceholder(const OutputNameView outputName) return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false); } - - - -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +void BasicDerivation::applyRewrites(const StringMap & rewrites) { - debug("Rewriting the derivation"); + if (rewrites.empty()) return; - for (auto & rewrite : rewrites) { + debug("rewriting the derivation"); + + for (auto & rewrite : rewrites) debug("rewriting %s as %s", rewrite.first, rewrite.second); - } - drv.builder = rewriteStrings(drv.builder, rewrites); - for (auto & arg : drv.args) { + builder = rewriteStrings(builder, rewrites); + for (auto & arg : args) arg = rewriteStrings(arg, rewrites); - } StringPairs newEnv; - for (auto & envVar : drv.env) { + for (auto & envVar : env) { auto envName = rewriteStrings(envVar.first, rewrites); auto envValue = rewriteStrings(envVar.second, rewrites); newEnv.emplace(envName, envValue); } - drv.env = newEnv; + env = std::move(newEnv); +} + +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +{ + drv.applyRewrites(rewrites); auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 40740d545..765b66ade 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -325,6 +325,12 @@ struct BasicDerivation static std::string_view nameFromPath(const StorePath & storePath); + /** + * Apply string rewrites to the `env`, `args` and `builder` + * fields. + */ + void applyRewrites(const StringMap & rewrites); + bool operator == (const BasicDerivation &) const = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. //auto operator <=> (const BasicDerivation &) const = default; From ced8d311a593fcf9c3823e4e118474ac132d8e60 Mon Sep 17 00:00:00 2001 From: Picnoir Date: Wed, 20 Nov 2024 13:33:00 +0100 Subject: [PATCH 043/243] gc: resume GC after a pathinuse error First the motivation: I recently faced a bug that I assume is coming from the topoSortPaths function where the GC was trying to delete a path having some alive referrers. I resolved this by manually deleting the faulty path referrers using nix-store --query --referrers. I sadly did not manage to reproduce this bug. This bug alone is not a big deal. However, this bug is triggering a cascading failure: invalidatePathChecked is throwing a PathInUse exception. This exception is not catched and fails the whole GC run. From there, the machine (a builder machine) was unable to GC its Nix store, which led to an almost full disk with no way to automatically delete the dead Nix paths. Instead, I think we should log the error for the specific store path we're trying to delete, specifying we can't delete this path because it still has referrers. Once we're done with logging that, the GC run should continue to delete the dead store paths it can delete. --- src/libstore/gc.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73195794a..45dfe4ad8 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -4,6 +4,7 @@ #include "finally.hh" #include "unix-domain-socket.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #if !defined(__linux__) // For shelling out to lsof @@ -763,13 +764,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } } } - for (auto & path : topoSortPaths(visited)) { if (!dead.insert(path).second) continue; if (shouldDelete) { - invalidatePathChecked(path); - deleteFromStore(path.to_string()); - referrersCache.erase(path); + try { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); + } catch (PathInUse &e) { + // If we end up here, it's likely a new occurence + // of https://github.com/NixOS/nix/issues/11923 + printError("BUG: %s", e.what()); + } } } }; From 1800853b2a450b8f80514d2f4acb8ab394a22705 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:03:58 +0300 Subject: [PATCH 044/243] fix(libexpr/eval-inline): get rid of references to nullptr env When diagnosing infinite recursion references to nullptr `Env` can be formed. This happens only with `ExprBlackHole` is evaluated, which always leads to `InfiniteRecursionError`. UBSAN log for one such case: ``` ../src/libexpr/eval-inline.hh:94:31: runtime error: reference binding to null pointer of type 'Env' SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ../src/libexpr/eval-inline.hh:94:31 in ``` --- src/libexpr/eval-inline.hh | 6 +++++- src/libexpr/eval.cc | 7 +++++-- src/libexpr/nixexpr.hh | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index d5ce238b2..631c0f396 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -87,11 +87,15 @@ void EvalState::forceValue(Value & v, const PosIdx pos) { if (v.isThunk()) { Env * env = v.payload.thunk.env; + assert(env || v.isBlackhole()); Expr * expr = v.payload.thunk.expr; try { v.mkBlackhole(); //checkInterrupt(); - expr->eval(*this, *env, v); + if (env) [[likely]] + expr->eval(*this, *env, v); + else + ExprBlackHole::throwInfiniteRecursionError(*this, v); } catch (...) { v.mkThunk(env, expr); tryFixupBlackHolePos(v, pos); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2fe2d5249..05f58957e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2052,9 +2052,12 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) state.mkPos(v, pos); } - -void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) +void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v) { + throwInfiniteRecursionError(state, v); +} + +[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value &v) { state.error("infinite recursion encountered") .atPos(v.determinePos(noPos)) .debugThrow(); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 948839bd9..2950ff1fd 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -468,6 +468,7 @@ struct ExprBlackHole : Expr void show(const SymbolTable & symbols, std::ostream & str) const override {} void eval(EvalState & state, Env & env, Value & v) override; void bindVars(EvalState & es, const std::shared_ptr & env) override {} + [[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v); }; extern ExprBlackHole eBlackHole; From ad7ad017ea5a7f7835148a03e81ef63a2689e8c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 16:35:47 +0100 Subject: [PATCH 045/243] EvalState::callPathFilter(): Remove unnecessary pathArg argument --- src/libexpr/eval.hh | 1 - src/libexpr/primops.cc | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 107334ffe..3ac3c8a8a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -819,7 +819,6 @@ public: bool callPathFilter( Value * filterFun, const SourcePath & path, - std::string_view pathArg, PosIdx pos); DocComment getDocCommentForPos(PosIdx pos); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 503632328..53bfce6c5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2438,7 +2438,6 @@ static RegisterPrimOp primop_toFile({ bool EvalState::callPathFilter( Value * filterFun, const SourcePath & path, - std::string_view pathArg, PosIdx pos) { auto st = path.lstat(); @@ -2446,7 +2445,7 @@ bool EvalState::callPathFilter( /* Call the filter function. The first argument is the path, the second is a string indicating the type of the file. */ Value arg1; - arg1.mkString(pathArg); + arg1.mkString(path.path.abs()); // assert that type is not "unknown" Value * args []{&arg1, fileTypeToString(*this, st.type)}; @@ -2489,7 +2488,7 @@ static void addPath( if (filterFun) filter = std::make_unique([&](const Path & p) { auto p2 = CanonPath(p); - return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + return state.callPathFilter(filterFun, {path.accessor, p2}, pos); }); std::optional expectedStorePath; From 5533b0c735bea0e7277b75ba1024bfbc2521c635 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:08:31 +0100 Subject: [PATCH 046/243] Move shebang flake tests into a separate test --- tests/functional/flakes/flakes.sh | 102 +----------------------- tests/functional/flakes/meson.build | 1 + tests/functional/flakes/shebang.sh | 117 ++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 101 deletions(-) create mode 100644 tests/functional/flakes/shebang.sh diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 5e901c5d1..2b503df02 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -74,100 +74,9 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" -#! nix --offline shell -#! nix --impure --expr `` -#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; -#! nix fooScript = flake.${builtins.currentSystem}.fooScript; -#! nix /* just a comment !@#$%^&*()__+ # */ -#! nix in fooScript -#! nix `` -#! nix --no-write-lock-file --command bash -set -ex -foo -echo "$@" -EOF -chmod +x $nonFlakeDir/shebang-inline-expr.sh - -cat > $nonFlakeDir/fooScript.nix <<"EOF" -let flake = (builtins.getFlake (toString ../flake1)).packages; - fooScript = flake.${builtins.currentSystem}.fooScript; - in fooScript -EOF - -cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" -#! nix --offline shell -#! nix --impure --file ./fooScript.nix -#! nix --no-write-lock-file --command bash -set -ex -foo -echo "$@" -EOF -chmod +x $nonFlakeDir/shebang-file.sh - # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -644,15 +553,6 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd # reference-lock-file can only be used if allow-dirty is set. expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock -# Test shebang -[[ $($nonFlakeDir/shebang.sh) = "foo" ]] -[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] -[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] -[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' - # Test that the --commit-lock-file-summary flag and its alias work cat > "$lockfileSummaryFlake/flake.nix" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + +[[ $($nonFlakeDir/shebang.sh) = "foo" ]] +[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From fd2df5f02f59f5f739c1e1931f8ff8c11a7720e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:23:20 +0100 Subject: [PATCH 047/243] Rename nonFlakeDir -> scriptDir --- tests/functional/flakes/shebang.sh | 50 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh index ac00ca764..23db19baa 100644 --- a/tests/functional/flakes/shebang.sh +++ b/tests/functional/flakes/shebang.sh @@ -7,15 +7,15 @@ TODO_NixOS requireGit flake1Dir=$TEST_ROOT/flake1 -nonFlakeDir=$TEST_ROOT/nonFlake +scriptDir=$TEST_ROOT/nonFlake createGitRepo "$flake1Dir" "" createSimpleGitFlake "$flake1Dir" nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" -mkdir -p "$nonFlakeDir" +mkdir -p "$scriptDir" -cat > "$nonFlakeDir/shebang.sh" < "$scriptDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $scriptDir/shebang-comments.sh < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $scriptDir/shebang-different-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $scriptDir/shebang-reject.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh < $scriptDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +cat >> $scriptDir/shebang-inline-expr.sh <<"EOF" #! nix --offline shell #! nix --impure --expr `` #! nix let flake = (builtins.getFlake (toString ../flake1)).packages; @@ -87,18 +87,18 @@ set -ex foo echo "$@" EOF -chmod +x $nonFlakeDir/shebang-inline-expr.sh +chmod +x $scriptDir/shebang-inline-expr.sh -cat > $nonFlakeDir/fooScript.nix <<"EOF" +cat > $scriptDir/fooScript.nix <<"EOF" let flake = (builtins.getFlake (toString ../flake1)).packages; fooScript = flake.${builtins.currentSystem}.fooScript; in fooScript EOF -cat > $nonFlakeDir/shebang-file.sh < $scriptDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +cat >> $scriptDir/shebang-file.sh <<"EOF" #! nix --offline shell #! nix --impure --file ./fooScript.nix #! nix --no-write-lock-file --command bash @@ -106,12 +106,12 @@ set -ex foo echo "$@" EOF -chmod +x $nonFlakeDir/shebang-file.sh +chmod +x $scriptDir/shebang-file.sh -[[ $($nonFlakeDir/shebang.sh) = "foo" ]] -[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] -[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] -[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' +[[ $($scriptDir/shebang.sh) = "foo" ]] +[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($scriptDir/shebang-comments.sh ) = "foo" ]] +[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]] +[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From e1cb905acae40b6bb55d30312ba9997c418eebf8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 18:42:33 +0100 Subject: [PATCH 048/243] Move --commit-lock-file-summary tests into a separate file --- .../flakes/commit-lock-file-summary.sh | 50 +++++++++++++++++++ tests/functional/flakes/flakes.sh | 40 +-------------- tests/functional/flakes/meson.build | 1 + 3 files changed, 52 insertions(+), 39 deletions(-) create mode 100644 tests/functional/flakes/commit-lock-file-summary.sh diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh new file mode 100644 index 000000000..21504eb21 --- /dev/null +++ b/tests/functional/flakes/commit-lock-file-summary.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +source ./common.sh + +TODO_NixOS + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake + +createGitRepo "$flake1Dir" "" +createSimpleGitFlake "$flake1Dir" +nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" + +createGitRepo "$lockfileSummaryFlake" "--initial-branch=main" + +# Test that the --commit-lock-file-summary flag and its alias work +cat > "$lockfileSummaryFlake/flake.nix" < "$lockfileSummaryFlake/flake.nix" < Date: Wed, 20 Nov 2024 18:51:23 +0100 Subject: [PATCH 049/243] Add a utility function for creating/registering a simple flake --- tests/functional/flakes/commit-lock-file-summary.sh | 8 +------- tests/functional/flakes/common.sh | 9 +++++++++ tests/functional/flakes/dubious-query.sh | 7 ++----- tests/functional/flakes/edit.sh | 7 +------ tests/functional/flakes/shebang.sh | 8 +------- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tests/functional/flakes/commit-lock-file-summary.sh b/tests/functional/flakes/commit-lock-file-summary.sh index 21504eb21..314d43ec3 100644 --- a/tests/functional/flakes/commit-lock-file-summary.sh +++ b/tests/functional/flakes/commit-lock-file-summary.sh @@ -4,15 +4,9 @@ source ./common.sh TODO_NixOS -requireGit +createFlake1 -flake1Dir=$TEST_ROOT/flake1 lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake - -createGitRepo "$flake1Dir" "" -createSimpleGitFlake "$flake1Dir" -nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" - createGitRepo "$lockfileSummaryFlake" "--initial-branch=main" # Test that the --commit-lock-file-summary flag and its alias work diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index cc9b2e466..0a020a1d7 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -38,12 +38,21 @@ EOF } createSimpleGitFlake() { + requireGit local flakeDir="$1" writeSimpleFlake "$flakeDir" git -C "$flakeDir" add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C "$flakeDir" commit -m 'Initial' } +# Create a simple Git flake and add it to the registry as "flake1". +createFlake1() { + flake1Dir="$TEST_ROOT/flake1" + createGitRepo "$flake1Dir" "" + createSimpleGitFlake "$flake1Dir" + nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" +} + writeDependentFlake() { local flakeDir="$1" cat > "$flakeDir/flake.nix" < "$scriptDir/shebang.sh" < Date: Wed, 20 Nov 2024 19:51:04 +0100 Subject: [PATCH 050/243] Move non-flake input tests into a separate file --- tests/functional/flakes/common.sh | 23 +++ tests/functional/flakes/flakes.sh | 166 ++------------------ tests/functional/flakes/meson.build | 1 + tests/functional/flakes/non-flake-inputs.sh | 126 +++++++++++++++ 4 files changed, 164 insertions(+), 152 deletions(-) create mode 100644 tests/functional/flakes/non-flake-inputs.sh diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 0a020a1d7..b1c3988e3 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -53,6 +53,29 @@ createFlake1() { nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" } +createFlake2() { + flake2Dir="$TEST_ROOT/flake 2" + percentEncodedFlake2Dir="$TEST_ROOT/flake%202" + + # Give one repo a non-main initial branch. + createGitRepo "$flake2Dir" "--initial-branch=main" + + cat > "$flake2Dir/flake.nix" < "$flakeDir/flake.nix" < "$flake2Dir/flake.nix" < "$flake3Dir/flake.nix" < "$nonFlakeDir/README.md" < "$flake3Dir/flake.nix" < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp "${config_nix}" "$flake3Dir" - -git -C "$flake3Dir" add flake.nix config.nix -git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file - -nix build -o "$TEST_ROOT/result" flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf "$TEST_HOME/.cache" -clearStore -mv "$flake2Dir" "$flake2Dir.tmp" -mv "$nonFlakeDir" "$nonFlakeDir.tmp" -nix build -o "$TEST_ROOT/result" flake3#sth -(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) -(! nix build -o "$TEST_ROOT/result" flake3#fnord) -mv "$flake2Dir.tmp" "$flake2Dir" -mv "$nonFlakeDir.tmp" "$nonFlakeDir" -nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord - # Test doing multiple `lookupFlake`s -nix build -o "$TEST_ROOT/result" flake4#xyzzy +nix build -o "$TEST_ROOT/result" flake3#xyzzy # Test 'nix flake update' and --override-flake. nix flake lock "$flake3Dir" @@ -330,53 +230,15 @@ nix flake lock "$flake3Dir" nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C "$flake3Dir" checkout -b removeXyzzy -rm "$flake3Dir/flake.nix" - -cat > "$flake3Dir/flake.nix" < \$out - ''; - }; - }; -} -EOF -nix flake lock "$flake3Dir" -git -C "$flake3Dir" add flake.nix flake.lock -git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' -git -C "$flake3Dir" checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) -nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth - # Testing the nix CLI nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry remove flake1 [[ $(nix registry list | wc -l) == 5 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 5 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 5 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 4 ]] # Test 'nix registry list' with a disabled global registry. nix registry add user-flake1 git+file://$flake1Dir @@ -386,7 +248,7 @@ nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in nix --flake-registry "" registry list | grepQuiet '^user' nix registry remove user-flake1 nix registry remove user-flake2 -[[ $(nix registry list | wc -l) == 5 ]] +[[ $(nix registry list | wc -l) == 4 ]] # Test 'nix flake clone'. rm -rf $TEST_ROOT/flake1-v2 diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index a646abc00..00060e3c9 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -26,6 +26,7 @@ suites += { 'dubious-query.sh', 'shebang.sh', 'commit-lock-file-summary.sh', + 'non-flake-inputs.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh new file mode 100644 index 000000000..66dc74e0c --- /dev/null +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +source ./common.sh + +createFlake1 +createFlake2 + +nonFlakeDir=$TEST_ROOT/nonFlake +createGitRepo "$nonFlakeDir" "" + +cat > "$nonFlakeDir/README.md" < "$flake3Dir/flake.nix" < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp "${config_nix}" "$flake3Dir" + +git -C "$flake3Dir" add flake.nix config.nix +git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file + +nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" + +nix build -o "$TEST_ROOT/result" flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf "$TEST_HOME/.cache" +clearStore +mv "$flake2Dir" "$flake2Dir.tmp" +mv "$nonFlakeDir" "$nonFlakeDir.tmp" +nix build -o "$TEST_ROOT/result" flake3#sth +(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! nix build -o "$TEST_ROOT/result" flake3#fnord) +mv "$flake2Dir.tmp" "$flake2Dir" +mv "$nonFlakeDir.tmp" "$nonFlakeDir" +nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C "$flake3Dir" checkout -b removeXyzzy +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < \$out + ''; + }; + }; +} +EOF +nix flake lock "$flake3Dir" +git -C "$flake3Dir" add flake.nix flake.lock +git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' +git -C "$flake3Dir" checkout master + +# Test whether fuzzy-matching works for registry entries. +nix registry add --registry "$registry" flake4 flake3 +(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) +nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth From db0525692dcb94e4f274e033091ad1ce54a30cf3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 21:07:22 +0100 Subject: [PATCH 051/243] Formatting --- src/libcmd/command.cc | 36 +++++++++++++++++++----------------- src/libcmd/command.hh | 5 +---- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0a3a516c2..86d13fab7 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -369,28 +369,30 @@ void MixEnvironment::setEnviron() return; } -void createOutLinks( - const std::filesystem::path & outLink, - const BuiltPaths & buildables, - LocalFSStore & store) +void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store) { for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - store.addPermRoot(bo.path, absPath(symlink.string())); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { auto symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store.addPermRoot(output.second, absPath(symlink.string())); - } + if (i) + symlink += fmt("-%d", i); + store.addPermRoot(bo.path, absPath(symlink.string())); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + auto symlink = outLink; + if (i) + symlink += fmt("-%d", i); + if (output.first != "out") + symlink += fmt("-%s", output.first); + store.addPermRoot(output.second, absPath(symlink.string())); + } + }, }, - }, buildable.raw()); + buildable.raw()); } } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 67c208d28..23529848f 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -372,9 +372,6 @@ void printClosureDiff( * Create symlinks prefixed by `outLink` to the store paths in * `buildables`. */ -void createOutLinks( - const std::filesystem::path & outLink, - const BuiltPaths & buildables, - LocalFSStore & store); +void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store); } From 2f24030bffa8f8f6769ac8ce0002d6c201312933 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Wed, 20 Nov 2024 13:23:02 -0800 Subject: [PATCH 052/243] Move bug report list to comment and make it more nix-specific --- .github/ISSUE_TEMPLATE/bug_report.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7ea82dcbc..a5005f8a0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -18,10 +18,13 @@ assignees: '' ## Steps To Reproduce -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error + ## Expected behavior From 671df02bf7d0cbf339d2626266260e0b7ccf5988 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 20:42:13 +0100 Subject: [PATCH 053/243] shellcheck --- tests/functional/flakes/non-flake-inputs.sh | 2 +- tests/functional/flakes/shebang.sh | 44 ++++++++++----------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index 66dc74e0c..c9c9803d3 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -71,7 +71,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" nix build -o "$TEST_ROOT/result" flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] +[[ $(cat "$TEST_ROOT/result") = FNORD ]] # Check whether flake input fetching is lazy: flake3#sth does not # depend on flake2, so this shouldn't fail. diff --git a/tests/functional/flakes/shebang.sh b/tests/functional/flakes/shebang.sh index 645b1fa38..576a62e2c 100644 --- a/tests/functional/flakes/shebang.sh +++ b/tests/functional/flakes/shebang.sh @@ -6,7 +6,7 @@ TODO_NixOS createFlake1 -scriptDir=$TEST_ROOT/nonFlake +scriptDir="$TEST_ROOT/nonFlake" mkdir -p "$scriptDir" cat > "$scriptDir/shebang.sh" < $scriptDir/shebang-comments.sh < "$scriptDir/shebang-comments.sh" < $scriptDir/shebang-comments.sh < $scriptDir/shebang-different-comments.sh < "$scriptDir/shebang-different-comments.sh" < $scriptDir/shebang-different-comments.sh < $scriptDir/shebang-reject.sh < "$scriptDir/shebang-reject.sh" < $scriptDir/shebang-reject.sh < $scriptDir/shebang-inline-expr.sh < "$scriptDir/shebang-inline-expr.sh" <> $scriptDir/shebang-inline-expr.sh <<"EOF" +cat >> "$scriptDir/shebang-inline-expr.sh" <<"EOF" #! nix --offline shell #! nix --impure --expr `` #! nix let flake = (builtins.getFlake (toString ../flake1)).packages; @@ -81,18 +79,18 @@ set -ex foo echo "$@" EOF -chmod +x $scriptDir/shebang-inline-expr.sh +chmod +x "$scriptDir/shebang-inline-expr.sh" -cat > $scriptDir/fooScript.nix <<"EOF" +cat > "$scriptDir/fooScript.nix" <<"EOF" let flake = (builtins.getFlake (toString ../flake1)).packages; fooScript = flake.${builtins.currentSystem}.fooScript; in fooScript EOF -cat > $scriptDir/shebang-file.sh < "$scriptDir/shebang-file.sh" <> $scriptDir/shebang-file.sh <<"EOF" +cat >> "$scriptDir/shebang-file.sh" <<"EOF" #! nix --offline shell #! nix --impure --file ./fooScript.nix #! nix --no-write-lock-file --command bash @@ -100,12 +98,12 @@ set -ex foo echo "$@" EOF -chmod +x $scriptDir/shebang-file.sh +chmod +x "$scriptDir/shebang-file.sh" -[[ $($scriptDir/shebang.sh) = "foo" ]] -[[ $($scriptDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] -[[ $($scriptDir/shebang-comments.sh ) = "foo" ]] -[[ "$($scriptDir/shebang-different-comments.sh)" = "$(cat $scriptDir/shebang-different-comments.sh)" ]] -[[ $($scriptDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] -[[ $($scriptDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] -expect 1 $scriptDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' +[[ $("$scriptDir/shebang.sh") = "foo" ]] +[[ $("$scriptDir/shebang.sh" "bar") = "foo"$'\n'"bar" ]] +[[ $("$scriptDir/shebang-comments.sh" ) = "foo" ]] +[[ "$("$scriptDir/shebang-different-comments.sh")" = "$(cat "$scriptDir/shebang-different-comments.sh")" ]] +[[ $("$scriptDir/shebang-inline-expr.sh" baz) = "foo"$'\n'"baz" ]] +[[ $("$scriptDir/shebang-file.sh" baz) = "foo"$'\n'"baz" ]] +expect 1 "$scriptDir/shebang-reject.sh" 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From e122acef97edd4a1190dba63e8f91b2b363a7df2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 20:55:45 +0100 Subject: [PATCH 054/243] Fix VM test --- tests/functional/flakes/non-flake-inputs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index c9c9803d3..f5e12cd01 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -2,6 +2,8 @@ source ./common.sh +TODO_NixOS + createFlake1 createFlake2 From 4a18c783853858b7bf897ac0a69baa3be7248237 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Nov 2024 21:26:20 +0100 Subject: [PATCH 055/243] flake_regressions: Pass -L to nix build --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb97fd211..9918875d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,4 +220,4 @@ jobs: path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh From f4f4b698f6bbd19cec767686eb3f017567d9fef2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 16:53:34 +0100 Subject: [PATCH 056/243] fetchTree: Don't crash if narHash is missing Fixes nix: ../src/libexpr/primops/fetchTree.cc:37: void nix::emitTreeAttrs(EvalState&, const StorePath&, const fetchers::Input&, Value&, bool, bool): Assertion `narHash' failed. on a lock file with an input that doesn't have a narHash. This can happen when using a lock file created by the lazy-trees branch. Cherry-picked from lazy-trees. --- src/libexpr/primops/fetchTree.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c207da8ad..47d1be1cc 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -33,9 +33,8 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); + if (auto narHash = input.getNarHash()) + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( From 965ca18db85638d6a04f4ecaff16b48fd661fe33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 18:44:57 +0100 Subject: [PATCH 057/243] Merge build-utils-meson/{diagnostics,threads} into build-utils-meson/common This reduces the amount of boilerplate. More importantly, it provides a place to add compiler flags (such as -O3) without having to add it to every subproject (and the risk of forgetting to include it). --- build-utils-meson/{diagnostics => common}/meson.build | 7 +++++++ build-utils-meson/threads/meson.build | 6 ------ src/libcmd/meson.build | 4 +--- src/libexpr-c/meson.build | 4 +--- src/libexpr-test-support/meson.build | 4 +--- src/libexpr-tests/meson.build | 4 +--- src/libexpr/meson.build | 4 +--- src/libfetchers-tests/meson.build | 4 +--- src/libfetchers/meson.build | 4 +--- src/libflake-tests/meson.build | 4 +--- src/libflake/meson.build | 4 +--- src/libmain-c/meson.build | 4 +--- src/libmain/meson.build | 4 +--- src/libstore-c/meson.build | 4 +--- src/libstore-test-support/meson.build | 4 +--- src/libstore-tests/meson.build | 4 +--- src/libstore/meson.build | 3 +-- src/libutil-c/meson.build | 4 +--- src/libutil-test-support/meson.build | 4 +--- src/libutil-tests/meson.build | 4 +--- src/libutil/meson.build | 3 +-- src/nix/meson.build | 4 +--- 22 files changed, 27 insertions(+), 64 deletions(-) rename build-utils-meson/{diagnostics => common}/meson.build (51%) delete mode 100644 build-utils-meson/threads/meson.build diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/common/meson.build similarity index 51% rename from build-utils-meson/diagnostics/meson.build rename to build-utils-meson/common/meson.build index 30eedfc13..67b6658f5 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/common/meson.build @@ -1,3 +1,10 @@ +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif + add_project_arguments( '-Wdeprecated-copy', '-Werror=suggest-override', diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build deleted file mode 100644 index 294160de1..000000000 --- a/build-utils-meson/threads/meson.build +++ /dev/null @@ -1,6 +0,0 @@ -# This is only conditional to work around -# https://github.com/mesonbuild/meson/issues/13293. It should be -# unconditional. -if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') - deps_private += dependency('threads') -endif diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index c484cf998..1f27c1614 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -30,8 +30,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -72,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'built-path.cc', diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 4160f0d5a..5bcca29e0 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -29,8 +29,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -55,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_expr.cc', diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index b9e7f390d..bdfd435a8 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -41,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/value/context.cc', diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 5a5c9f1d4..b50c18c9c 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -51,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 4d8a38b43..28318579e 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - boost = dependency( 'boost', modules : ['container', 'context'], @@ -79,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') parser_tab = custom_target( input : 'parser.y', diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index d948dbad6..fdab6ba6c 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -24,8 +24,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index ff638578d..07a1178cc 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -43,7 +41,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'attrs.cc', diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 592a7493b..c0a9b8847 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -24,8 +24,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'flakeref.cc', diff --git a/src/libflake/meson.build b/src/libflake/meson.build index d2bb179df..2c1a70a18 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -41,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'flake/config.cc', diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 0ec0e3f6d..3cb1e4baa 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -29,8 +29,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -55,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_main.cc', diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 7fcadf06d..6c6298e2b 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -26,8 +26,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - pubsetbuf_test = ''' #include @@ -60,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'common-args.cc', diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index d4f86eeff..44b5fe11d 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -27,8 +27,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -51,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_store.cc', diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 98ec9882e..f8308e7bb 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -38,7 +36,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/derived-path.cc', diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index f4f67d73a..fc9152f2f 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -52,7 +50,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'common-protocol.cc', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 101879e90..f836b8d4f 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -82,7 +82,6 @@ if host_machine.system() == 'windows' endif subdir('build-utils-meson/libatomic') -subdir('build-utils-meson/threads') boost = dependency( 'boost', @@ -180,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'binary-cache-store.cc', diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 3d5a0b9c2..d44453676 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -47,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'nix_api_util.cc', diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index c5e1ba80b..fa1df7320 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -23,8 +23,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -35,7 +33,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'tests/hash.cc', diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 5c3b5e5a3..f59350774 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -25,8 +25,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -44,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'args.cc', diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a6dc86394..11b4ea592 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -54,7 +54,6 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) subdir('build-utils-meson/libatomic') -subdir('build-utils-meson/threads') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -121,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') sources = files( 'archive.cc', diff --git a/src/nix/meson.build b/src/nix/meson.build index 60ee48035..5c70c8216 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -32,8 +32,6 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') -subdir('build-utils-meson/threads') - subdir('build-utils-meson/export-all-symbols') subdir('build-utils-meson/windows-version') @@ -65,7 +63,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/common') subdir('build-utils-meson/generate-header') nix_sources = [config_h] + files( From ed120a61abc35b2a0a782dfce3935013cc108d50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Nov 2024 19:16:27 +0100 Subject: [PATCH 058/243] Use -O3 again This was lost in the switch to the new build system. -O3 provides around a 10% performance gain compared to -O2, see e.g. nix-env.qaAggressive.time in https://hydra.nixos.org/job/nix/master/metrics.nixpkgs#tabs-charts. --- build-utils-meson/common/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-utils-meson/common/meson.build b/build-utils-meson/common/meson.build index 67b6658f5..f0322183e 100644 --- a/build-utils-meson/common/meson.build +++ b/build-utils-meson/common/meson.build @@ -16,3 +16,7 @@ add_project_arguments( '-Wno-deprecated-declarations', language : 'cpp', ) + +if get_option('buildtype') not in ['debug'] + add_project_arguments('-O3', language : 'cpp') +endif From ba074465ba249da4df0752d0dc53529d0ab1a60d Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Thu, 21 Nov 2024 20:08:13 +0000 Subject: [PATCH 059/243] doc: Clarify that nix-shell still uses shell from host environment (#8809) * doc: Clarify that nix-shell still uses shell from host environment * doc: Fix NIX_BUILD_SHELL description * doc: Add anchor and link to NIX_BUILD_SHELL * doc: Add example of default shell trickiness Co-authored-by: Valentin Gagarin --- doc/manual/source/command-ref/nix-shell.md | 31 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/doc/manual/source/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md index 69a711bd5..e95db9bea 100644 --- a/doc/manual/source/command-ref/nix-shell.md +++ b/doc/manual/source/command-ref/nix-shell.md @@ -88,7 +88,9 @@ All options not listed here are passed to `nix-store cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + retained. Note that the shell used to run commands is obtained from + [`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL) / `` from + `NIX_PATH`, and therefore not affected by `--pure`. - `--packages` / `-p` *packages*… @@ -112,11 +114,30 @@ All options not listed here are passed to `nix-store # Environment variables -- `NIX_BUILD_SHELL` +- [`NIX_BUILD_SHELL`](#env-NIX_BUILD_SHELL) - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. + Shell used to start the interactive environment. + Defaults to the `bash` from `bashInteractive` found in ``, falling back to the `bash` found in `PATH` if not found. + + > **Note** + > + > The shell obtained using this method may not necessarily be the same as any shells requested in *path*. + + + + > **Example + > + > Despite `--pure`, this invocation will not result in a fully reproducible shell environment: + > + > ```nix + > #!/usr/bin/env -S nix-shell --pure + > let + > pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/854fdc68881791812eddd33b2fed94b954979a8e.tar.gz") {}; + > in + > pkgs.mkShell { + > buildInputs = pkgs.bashInteractive; + > } + > ``` {{#include ./env-common.md}} From ebb19cc1cdc2f04c291e2e8e34f434e5defa5bbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2024 09:14:01 +0100 Subject: [PATCH 060/243] Drop std::make_pair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- src/libflake/flake/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index ed47ad737..cdcdcf87f 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -78,7 +78,7 @@ static std::pair fromParsedURL( std::string fragment; std::swap(fragment, parsedURL.fragment); - return std::make_pair(FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment); + return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment}; } std::pair parsePathFlakeRefWithFragment( From 756758d968375c3f346412f8f01c82a9178689fb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:05:52 +0300 Subject: [PATCH 061/243] chore: get rid of dead code and unused variables where appropriate Looks like some cruft has been left over from previous refactorings. This removes dead variables, which should not have side effects in their constructors. In cases where the variable initialization has a purpose [[maybe_unused]] is inserted to silence compiler warnings. --- src/libexpr/attr-path.cc | 1 - src/libexpr/primops.cc | 2 -- src/libfetchers/git.cc | 2 -- src/libflake/flake/flakeref.cc | 2 -- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/export-import.cc | 5 ----- src/libstore/unix/build/local-derivation-goal.cc | 2 +- src/libutil-tests/nix_api_util.cc | 1 - src/libutil/serialise.cc | 3 +-- src/libutil/terminal.cc | 2 +- src/nix/search.cc | 1 - src/nix/sigs.cc | 1 - 12 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2f67260c5..822ec7620 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -129,7 +129,6 @@ std::pair findPackageFilename(EvalState & state, Value & v try { auto colon = fn.rfind(':'); if (colon == std::string::npos) fail(); - std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 53bfce6c5..5d2f75373 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -66,14 +66,12 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - auto ctxS = store->printStorePath(o.path); ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ - auto ctxS = store->printStorePath(d.drvPath); ensureValid(d.drvPath); if (maybePathsOut) maybePathsOut->emplace(d.drvPath); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 99d91919e..a6883a2d3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -514,8 +514,6 @@ struct GitInputScheme : InputScheme auto origRev = input.getRev(); - std::string name = input.getName(); - auto originalRef = input.getRef(); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index cdcdcf87f..9616fe0ea 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -257,8 +257,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - std::smatch match; - if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 794be1568..bf1a25db1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1229,7 +1229,7 @@ HookReply DerivationGoal::tryBuildHook() hook->toHook.writeSide.close(); /* Create the log file and pipe. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); std::set fds; fds.insert(hook->fromHook.readSide.get()); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb36c0c1b..1c62cdfad 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -13,14 +13,9 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - std::string doneLabel("paths exported"); - //logger->incExpected(doneLabel, sorted.size()); - for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); - //logger->incProgress(doneLabel); } sink << 0; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index dcfaadeef..06a2f85be 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -883,7 +883,7 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index b36f71042..7b77bd87f 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -136,7 +136,6 @@ TEST_F(nix_api_util_context, nix_err_name) // no error EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); - std::string err_msg_ref; try { throw nix::Error("testing error"); } catch (...) { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 168d2ed32..381e7ae38 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -90,7 +90,6 @@ void Source::operator () (std::string_view data) void Source::drainInto(Sink & sink) { - std::string s; std::array buf; while (true) { size_t n; @@ -427,7 +426,7 @@ Error readError(Source & source) auto type = readString(source); assert(type == "Error"); auto level = (Verbosity) readInt(source); - auto name = readString(source); // removed + [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); ErrorInfo info { .level = level, diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index db7a6fcd1..4c127ddb0 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -26,7 +26,7 @@ bool isTTY() std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) { - std::string t, e; + std::string t; size_t w = 0; auto i = s.begin(); diff --git a/src/nix/search.cc b/src/nix/search.cc index c8d0b9e96..30b96c500 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -161,7 +161,6 @@ struct CmdSearch : InstallableValueCommand, MixJSON {"description", description}, }; } else { - auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 2afe4b267..134d4f34a 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -41,7 +41,6 @@ struct CmdCopySigs : StorePathsCommand ThreadPool pool; - std::string doneLabel = "done"; std::atomic added{0}; //logger->setExpected(doneLabel, storePaths.size()); From 00b99b8bc0aa7f5f0a1a04b9704c2120e5eca6db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2024 16:23:34 +0100 Subject: [PATCH 062/243] Remove FIXME --- src/libflake/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e49333f25..8e381c0d2 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -470,7 +470,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix }); } } From 09ddc34b62bf762cbe7e0a9adce4bdea9ff7fc6a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:01:11 +0300 Subject: [PATCH 063/243] refactor(libfetchers/registry): use standard remove_if + erase Get rid of this fixme. This does not appear to be used anywhere in the nix codebase itself. Not sure why the comment mentioned C++20 erase member function with predicate, but iterator-based algorithms are also fine. --- src/libfetchers/registry.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 7f7a09053..c761028ab 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -94,12 +94,9 @@ void Registry::add( void Registry::remove(const Input & input) { - // FIXME: use C++20 std::erase. - for (auto i = entries.begin(); i != entries.end(); ) - if (i->from == input) - i = entries.erase(i); - else - ++i; + entries.erase( + std::remove_if(entries.begin(), entries.end(), [&](const Entry & entry) { return entry.from == input; }), + entries.end()); } static Path getSystemRegistryPath() From fbffd47fb715396f47be21ae0ed1a7f484852b4b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:35:47 +0300 Subject: [PATCH 064/243] build(lib{expr,store,util}-test-support): depend on -c libraries Since lib{expr,store,util}-test-support subprojects define nix_api_* helpers for testing nix c bindings, they need to publicly depend on -c counterparts. This makes their headers self-sufficient and does not rely on the -tests to add necessary dependencies. --- src/libexpr-test-support/meson.build | 1 + src/libexpr-test-support/package.nix | 2 ++ src/libstore-test-support/meson.build | 1 + src/libstore-test-support/package.nix | 2 ++ src/libutil-test-support/meson.build | 1 + src/libutil-test-support/package.nix | 2 ++ 6 files changed, 9 insertions(+) diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index bdfd435a8..33d9e17a6 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -24,6 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-test-support'), dependency('nix-expr'), + dependency('nix-expr-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index bcf6118e0..7e92d145f 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -4,6 +4,7 @@ , nix-store-test-support , nix-expr +, nix-expr-c , rapidcheck @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-store-test-support nix-expr + nix-expr-c rapidcheck ]; diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index f8308e7bb..1f230914f 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -22,6 +22,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-test-support'), dependency('nix-store'), + dependency('nix-store-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 48f8b5e6b..2543049fe 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -4,6 +4,7 @@ , nix-util-test-support , nix-store +, nix-store-c , rapidcheck @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util-test-support nix-store + nix-store-c rapidcheck ]; diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index fa1df7320..4afed01ca 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ dependency('nix-util'), + dependency('nix-util-c'), ] subdir('build-utils-meson/subprojects') diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 2525e1602..c403e762c 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -3,6 +3,7 @@ , mkMesonLibrary , nix-util +, nix-util-c , rapidcheck @@ -33,6 +34,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util + nix-util-c rapidcheck ]; From 4145d18435f5e7073f0c99f813d45dd9058430dd Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sat, 23 Nov 2024 08:36:51 +0300 Subject: [PATCH 065/243] Rename install-nix-from-closure.sh into install-nix-from-tarball.sh Because it is only used as /install script from tarball. --- maintainers/flake-module.nix | 2 +- scripts/binary-tarball.nix | 2 +- ...{install-nix-from-closure.sh => install-nix-from-tarball.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scripts/{install-nix-from-closure.sh => install-nix-from-tarball.sh} (100%) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index fdb031302..ba6cd2816 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -496,7 +496,7 @@ ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-nix-from-tarball\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 104189b0c..671c8e96e 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -23,7 +23,7 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + substitute ${./install-nix-from-tarball.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-tarball.sh similarity index 100% rename from scripts/install-nix-from-closure.sh rename to scripts/install-nix-from-tarball.sh From 82a23d9b6b96bf08e7c28008fcc346c0bdb671be Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:24:20 +0100 Subject: [PATCH 066/243] libexpr-c: Add nix_eval_state_builder --- src/libexpr-c/nix_api_expr.cc | 79 ++++++++++++++++++++++++--- src/libexpr-c/nix_api_expr.h | 65 +++++++++++++++++++++- src/libexpr-c/nix_api_expr_internal.h | 11 ++++ src/libexpr-tests/nix_api_expr.cc | 37 +++++++++++++ 4 files changed, 183 insertions(+), 9 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 6144a7986..27a70afb6 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -6,6 +6,7 @@ #include "eval-gc.hh" #include "globals.hh" #include "eval-settings.hh" +#include "ref.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -93,7 +94,46 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val NIXC_CATCH_ERRS } -EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // Allocate ahead of time, because .settings needs self-reference + void * p = ::operator new( + sizeof(nix_eval_state_builder), + static_cast(alignof(nix_eval_state_builder))); + auto * p2 = static_cast(p); + new (p) nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = true, + }; + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_eval_state_builder_free(nix_eval_state_builder * builder) +{ + delete builder; +} + +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // TODO: load in one go? + builder->settings.readOnlyMode = nix::settings.readOnlyMode; + loadConfFile(builder->settings); + loadConfFile(builder->fetchSettings); + } + NIXC_CATCH_ERRS +} + +nix_err nix_eval_state_builder_set_lookup_path(nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath_c) { if (context) context->last_err_code = NIX_OK; @@ -102,28 +142,51 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c if (lookupPath_c != nullptr) for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); + builder->lookupPath = nix::LookupPath::parse(lookupPath); + } + NIXC_CATCH_ERRS +} +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // Allocate ahead of time, because .state init needs self-reference void * p = ::operator new( sizeof(EvalState), static_cast(alignof(EvalState))); auto * p2 = static_cast(p); new (p) EvalState { - .fetchSettings = nix::fetchers::Settings{}, - .settings = nix::EvalSettings{ - nix::settings.readOnlyMode, - }, + .fetchSettings = std::move(builder->fetchSettings), + .settings = std::move(builder->settings), .state = nix::EvalState( - nix::LookupPath::parse(lookupPath), - store->ptr, + builder->lookupPath, + builder->store, p2->fetchSettings, p2->settings), }; - loadConfFile(p2->settings); return p2; } NIXC_CATCH_ERRS_NULL } +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + auto builder = nix_eval_state_builder_new(context, store); + if (builder == nullptr) + return nullptr; + + if (nix_eval_state_builder_load(context, builder) != NIX_OK) + return nullptr; + + if (nix_eval_state_builder_set_lookup_path(context, builder, lookupPath_c) + != NIX_OK) + return nullptr; + + return nix_eval_state_build(context, builder); +} + void nix_state_free(EvalState * state) { delete state; diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index e680f5ff1..f8d181452 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -30,6 +30,11 @@ extern "C" { // cffi start // Type definitions +/** + * @brief Builder for EvalState + */ +typedef struct nix_eval_state_builder nix_eval_state_builder; + /** * @brief Represents a state of the Nix language evaluator. * @@ -174,12 +179,70 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** - * @brief Create a new Nix language evaluator state. + * @brief Create a new nix_eval_state_builder + * + * The settings are initialized to their default value. + * Values can be sourced elsewhere with nix_eval_state_builder_load. + * + * @param[out] context Optional, stores error information + * @param[in] store The Nix store to use. + * @return A new nix_eval_state_builder or NULL on failure. + */ +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store); + +/** + * @brief Read settings from the ambient environment + * + * Settings are sourced from environment variables and configuration files, + * as documented in the Nix manual. + * + * @param[out] context Optional, stores error information + * @param[out] builder The builder to modify. + * @return NIX_OK if successful, an error code otherwise. + */ +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Set the lookup path for `<...>` expressions + * + * @param[in] context Optional, stores error information + * @param[in] builder The builder to modify. + * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. + */ +nix_err nix_eval_state_builder_set_lookup_path( + nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath); + +/** + * @brief Create a new Nix language evaluator state + * + * Remember to nix_eval_state_builder_free after building the state. + * + * @param[out] context Optional, stores error information + * @param[in] builder The builder to use and free + * @return A new Nix state or NULL on failure. + * @see nix_eval_state_builder_new, nix_eval_state_builder_free + */ +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Free a nix_eval_state_builder + * + * Does not fail. + * + * @param[in] builder The builder to free. + */ +void nix_eval_state_builder_free(nix_eval_state_builder * builder); + +/** + * @brief Create a new Nix language evaluator state + * + * For more control, use nix_eval_state_builder * * @param[out] context Optional, stores error information * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. * @return A new Nix state or NULL on failure. + * @see nix_state_builder_new */ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 12f24b6eb..f59664011 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -6,6 +6,17 @@ #include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" +#include "search-path.hh" + +struct nix_eval_state_builder +{ + nix::ref store; + nix::EvalSettings settings; + nix::fetchers::Settings fetchSettings; + nix::LookupPath lookupPath; + // TODO: make an EvalSettings setting own this instead? + bool readOnlyMode; +}; struct EvalState { diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index b37ac44b3..5ed78d2fc 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -7,12 +7,49 @@ #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" +#include "file-system.hh" #include #include namespace nixC { +TEST_F(nix_api_store_test, nix_eval_state_lookup_path) +{ + auto tmpDir = nix::createTempDir(); + auto delTmpDir = std::make_unique(tmpDir, true); + auto nixpkgs = tmpDir + "/pkgs"; + auto nixos = tmpDir + "/cfg"; + std::filesystem::create_directories(nixpkgs); + std::filesystem::create_directories(nixos); + + std::string nixpkgsEntry = "nixpkgs=" + nixpkgs; + std::string nixosEntry = "nixos-config=" + nixos; + const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr}; + + auto builder = nix_eval_state_builder_new(ctx, store); + assert_ctx_ok(); + + ASSERT_EQ(NIX_OK, nix_eval_state_builder_set_lookup_path(ctx, builder, lookupPath)); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + + nix_eval_state_builder_free(builder); + + Value * value = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.seq ", ".", value); + assert_ctx_ok(); + + ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH); + assert_ctx_ok(); + + auto pathStr = nix_get_path_string(ctx, value); + assert_ctx_ok(); + ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str())); +} + TEST_F(nix_api_expr_test, nix_expr_eval_from_string) { nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); From 1bd75178017102e098e7b9a6aa2ddf858b486b53 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:27:17 +0100 Subject: [PATCH 067/243] Doc nix_get_path_string --- src/libexpr-c/nix_api_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 8a0813ebe..711b0adbc 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -213,7 +213,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_ /** @brief Get path as string * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return string + * @return string, if the type is NIX_TYPE_PATH * @return NULL in case of error. */ const char * nix_get_path_string(nix_c_context * context, const nix_value * value); From f06f611ff3f1e7beee51171b3ab13d4883e187f3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 17:26:26 +0100 Subject: [PATCH 068/243] refactor: Extract unsafe_new_with_self --- src/libexpr-c/nix_api_expr.cc | 69 +++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 27a70afb6..a024248cd 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,6 +19,29 @@ # include #endif +/** + * @brief Allocate and initialize using self-reference + * + * This allows a brace initializer to reference the object being constructed. + * + * @warning Use with care, as the pointer points to an object that is not fully constructed yet. + * + * @tparam T Type to allocate + * @tparam F A function type for `init`, taking a T* and returning the initializer for T + * @param init Function that takes a T* and returns the initializer for T + * @return Pointer to allocated and initialized object + */ +template +static T * unsafe_new_with_self(F && init) +{ + // Allocate + void * p = ::operator new( + sizeof(T), + static_cast(alignof(T))); + // Initialize with placement new + return new (p) T(init(static_cast(p))); +} + nix_err nix_libexpr_init(nix_c_context * context) { if (context) @@ -99,18 +122,14 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - // Allocate ahead of time, because .settings needs self-reference - void * p = ::operator new( - sizeof(nix_eval_state_builder), - static_cast(alignof(nix_eval_state_builder))); - auto * p2 = static_cast(p); - new (p) nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ p2->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - return p2; + return unsafe_new_with_self([&](auto * self) { + return nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = true, + }; + }); } NIXC_CATCH_ERRS_NULL } @@ -152,21 +171,17 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder if (context) context->last_err_code = NIX_OK; try { - // Allocate ahead of time, because .state init needs self-reference - void * p = ::operator new( - sizeof(EvalState), - static_cast(alignof(EvalState))); - auto * p2 = static_cast(p); - new (p) EvalState { - .fetchSettings = std::move(builder->fetchSettings), - .settings = std::move(builder->settings), - .state = nix::EvalState( - builder->lookupPath, - builder->store, - p2->fetchSettings, - p2->settings), - }; - return p2; + return unsafe_new_with_self([&](auto * self) { + return EvalState{ + .fetchSettings = std::move(builder->fetchSettings), + .settings = std::move(builder->settings), + .state = nix::EvalState( + builder->lookupPath, + builder->store, + self->fetchSettings, + self->settings), + }; + }); } NIXC_CATCH_ERRS_NULL } From 4eecf3c20ab454b4427363a276d757298f9220dc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 22 Nov 2024 16:18:55 +0100 Subject: [PATCH 069/243] Add nix-flake-c, nix_flake_init_global, nix_flake_settings_new --- meson.build | 1 + packaging/components.nix | 1 + packaging/everything.nix | 4 + src/external-api-docs/doxygen.cfg.in | 1 + src/external-api-docs/package.nix | 1 + src/libflake-c/.version | 1 + src/libflake-c/build-utils-meson | 1 + src/libflake-c/meson.build | 93 ++++++++++++++++++++++++ src/libflake-c/nix_api_flake.cc | 32 ++++++++ src/libflake-c/nix_api_flake.h | 46 ++++++++++++ src/libflake-c/nix_api_flake_internal.hh | 9 +++ src/libflake-c/package.nix | 60 +++++++++++++++ src/libflake-tests/meson.build | 3 + src/libflake-tests/nix_api_flake.cc | 51 +++++++++++++ src/libflake-tests/package.nix | 3 + 15 files changed, 307 insertions(+) create mode 120000 src/libflake-c/.version create mode 120000 src/libflake-c/build-utils-meson create mode 100644 src/libflake-c/meson.build create mode 100644 src/libflake-c/nix_api_flake.cc create mode 100644 src/libflake-c/nix_api_flake.h create mode 100644 src/libflake-c/nix_api_flake_internal.hh create mode 100644 src/libflake-c/package.nix create mode 100644 src/libflake-tests/nix_api_flake.cc diff --git a/meson.build b/meson.build index 8985b631e..49adf9832 100644 --- a/meson.build +++ b/meson.build @@ -34,6 +34,7 @@ endif subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libflake-c') subproject('libmain-c') # Language Bindings diff --git a/packaging/components.nix b/packaging/components.nix index c29e04ae9..e1f661be8 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -44,6 +44,7 @@ in nix-expr-tests = callPackage ../src/libexpr-tests/package.nix { }; nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-c = callPackage ../src/libflake-c/package.nix { }; nix-flake-tests = callPackage ../src/libflake-tests/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; diff --git a/packaging/everything.nix b/packaging/everything.nix index b09b9d2a9..0b04d2c6d 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -19,6 +19,7 @@ nix-expr-tests, nix-flake, + nix-flake-c, nix-flake-tests, nix-main, @@ -53,6 +54,7 @@ let nix-expr-c nix-fetchers nix-flake + nix-flake-c nix-main nix-main-c nix-store @@ -86,6 +88,7 @@ let "nix-expr-c" "nix-fetchers" "nix-flake" + "nix-flake-c" "nix-main" "nix-main-c" "nix-store" @@ -169,6 +172,7 @@ in nix-expr nix-expr-c nix-flake + nix-flake-c nix-main nix-main-c ; diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 8e235dae5..3af2f5b81 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -40,6 +40,7 @@ GENERATE_LATEX = NO INPUT = \ @src@/src/libutil-c \ @src@/src/libexpr-c \ + @src@/src/libflake-c \ @src@/src/libstore-c \ @src@/src/external-api-docs/README.md diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index 0c592955a..57c5138cf 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libflake-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; diff --git a/src/libflake-c/.version b/src/libflake-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson new file mode 120000 index 000000000..91937f183 --- /dev/null +++ b/src/libflake-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build new file mode 100644 index 000000000..00d9650e7 --- /dev/null +++ b/src/libflake-c/meson.build @@ -0,0 +1,93 @@ +project('nix-flake-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), + dependency('nix-expr-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-flake.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + # not generated (yet?) + # '-include', 'config-flake.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + '-include', 'config-flake.h', + language : 'cpp', +) + +subdir('build-utils-meson/common') + +sources = files( + 'nix_api_flake.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_flake.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_flake.h') + +subdir('build-utils-meson/export-all-symbols') +subdir('build-utils-meson/windows-version') + +this_library = library( + 'nixflakec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc new file mode 100644 index 000000000..17cf6572d --- /dev/null +++ b/src/libflake-c/nix_api_flake.cc @@ -0,0 +1,32 @@ +#include "nix_api_flake.h" +#include "nix_api_flake_internal.hh" +#include "nix_api_util_internal.h" + +#include "flake/flake.hh" + +nix_flake_settings * nix_flake_settings_new(nix_c_context * context) +{ + try { + auto settings = nix::make_ref(); + return new nix_flake_settings{settings}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_flake_settings_free(nix_flake_settings * settings) +{ + delete settings; +} + +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings) +{ + static std::shared_ptr registeredSettings; + try { + if (registeredSettings) + throw nix::Error("nix_flake_init_global already initialized"); + + registeredSettings = settings->settings; + nix::flake::initLib(*registeredSettings); + } + NIXC_CATCH_ERRS +} diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h new file mode 100644 index 000000000..80051298d --- /dev/null +++ b/src/libflake-c/nix_api_flake.h @@ -0,0 +1,46 @@ +#ifndef NIX_API_FLAKE_H +#define NIX_API_FLAKE_H +/** @defgroup libflake libflake + * @brief Bindings to the Nix Flakes library + * + * @{ + */ +/** @file + * @brief Main entry for the libflake C bindings + */ + +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +typedef struct nix_flake_settings nix_flake_settings; + +// Function prototypes +/** + * Create a nix_flake_settings initialized with default values. + * @param[out] context Optional, stores error information + * @return A new nix_flake_settings or NULL on failure. + * @see nix_flake_settings_free + */ +nix_flake_settings * nix_flake_settings_new(nix_c_context * context); + +/** + * @brief Release the resources associated with a nix_flake_settings. + */ +void nix_flake_settings_free(nix_flake_settings * settings); + +/** + * @brief Register Flakes support process-wide. + */ +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh new file mode 100644 index 000000000..4c154a342 --- /dev/null +++ b/src/libflake-c/nix_api_flake_internal.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "ref.hh" +#include "flake/settings.hh" + +struct nix_flake_settings +{ + nix::ref settings; +}; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix new file mode 100644 index 000000000..a70cbf94e --- /dev/null +++ b/src/libflake-c/package.nix @@ -0,0 +1,60 @@ +{ lib +, stdenv +, mkMesonLibrary + +, nix-store-c +, nix-expr-c +, nix-flake + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonLibrary (finalAttrs: { + pname = "nix-flake-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + propagatedBuildInputs = [ + nix-expr-c + nix-store-c + nix-flake + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c0a9b8847..c494c414e 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -19,6 +19,7 @@ subdir('build-utils-meson/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), dependency('nix-flake'), + dependency('nix-flake-c'), ] deps_public_maybe_subproject = [ ] @@ -46,6 +47,7 @@ subdir('build-utils-meson/common') sources = files( 'flakeref.cc', + 'nix_api_flake.cc', 'url-name.cc', ) @@ -68,6 +70,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'NIX_CONFIG': 'extra-experimental-features = flakes', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc new file mode 100644 index 000000000..21109d181 --- /dev/null +++ b/src/libflake-tests/nix_api_flake.cc @@ -0,0 +1,51 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "nix_api_flake.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include +#include + +namespace nixC { + +TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists) +{ + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_flake_init_global(ctx, settings); + assert_ctx_ok(); + + nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store); + ASSERT_NE(nullptr, builder); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + ASSERT_NE(nullptr, state); + + nix_eval_state_builder_free(builder); + + auto value = nix_alloc_value(ctx, state); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, err); + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value)); +} + +} // namespace nixC diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 67e716979..b3a8ac466 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -4,6 +4,7 @@ , mkMesonExecutable , nix-flake +, nix-flake-c , nix-expr-test-support , rapidcheck @@ -38,6 +39,7 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ nix-flake + nix-flake-c nix-expr-test-support rapidcheck gtest @@ -67,6 +69,7 @@ mkMesonExecutable (finalAttrs: { mkdir -p "$HOME" '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export NIX_CONFIG="extra-experimental-features = flakes" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); From d004c524b84651c2eebb5bbb55f6a3a8324437e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 24 Nov 2024 13:52:40 +0100 Subject: [PATCH 070/243] test: Change FAIL to throw [FAIL()] is a macro with `return`, making it unsuitable for helpers. This uses std::runtime_error, because gtest does not seem to provide an exception type of its own for this purpose. [AssertionException] is for a different use case. [FAIL()]: https://google.github.io/googletest/reference/assertions.html#FAIL [AssertionException]: https://github.com/google/googletest/blob/35d0c365609296fa4730d62057c487e3cfa030ff/docs/reference/testing.md#assertionexception-assertionexception --- src/libutil-test-support/tests/gtest-with-params.hh | 2 +- src/libutil-test-support/tests/nix_api_util.hh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil-test-support/tests/gtest-with-params.hh b/src/libutil-test-support/tests/gtest-with-params.hh index d72aec4fd..a6e23ad89 100644 --- a/src/libutil-test-support/tests/gtest-with-params.hh +++ b/src/libutil-test-support/tests/gtest-with-params.hh @@ -40,7 +40,7 @@ void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) } else { std::ostringstream ss; printResultMessage(result, ss); - FAIL() << ss.str() << std::endl; + throw std::runtime_error(ss.str()); } } } diff --git a/src/libutil-test-support/tests/nix_api_util.hh b/src/libutil-test-support/tests/nix_api_util.hh index efd200116..006dc497c 100644 --- a/src/libutil-test-support/tests/nix_api_util.hh +++ b/src/libutil-test-support/tests/nix_api_util.hh @@ -26,14 +26,13 @@ protected: inline void assert_ctx_ok() { - if (nix_err_code(ctx) == NIX_OK) { return; } unsigned int n; const char * p = nix_err_msg(nullptr, ctx, &n); std::string msg(p, n); - FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + throw std::runtime_error(std::string("nix_err_code(ctx) != NIX_OK, message: ") + msg); } inline void assert_ctx_err() @@ -41,7 +40,7 @@ protected: if (nix_err_code(ctx) != NIX_OK) { return; } - FAIL() << "Got NIX_OK, but expected an error!"; + throw std::runtime_error("Got NIX_OK, but expected an error!"); } }; From 6db6b269ed70788314209d35499812c90949057f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Nov 2024 01:16:02 +0100 Subject: [PATCH 071/243] .github/ci: Set max-jobs to 1, to reduce peak memory usage --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9918875d9..be96bb484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,9 @@ jobs: - uses: cachix/install-nix-action@v30 with: # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" + extra_nix_config: | + sandbox = true + max-jobs = 1 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' From 6502dc4d6af5baad369578ee0b4d2e1295d199a7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Nov 2024 12:06:54 +0100 Subject: [PATCH 072/243] ci(Mergify): configuration update Signed-off-by: Robert Hensing --- .mergify.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index c297d3d5e..86623a138 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -90,3 +90,13 @@ pull_request_rules: - "2.24-maintenance" labels: - merge-queue + + - name: backport patches to 2.25 + conditions: + - label=backport 2.25-maintenance + actions: + backport: + branches: + - "2.25-maintenance" + labels: + - merge-queue From 3fb7481e64fc22313211c6c4ea79ac314457f81b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 24 Nov 2024 11:17:17 +0100 Subject: [PATCH 073/243] source-accessor: fix case where normalization goes beyond root fixes https://github.com/NixOS/nix/issues/11936 --- src/libutil/source-accessor.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e797951c7..d3e304f74 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -84,9 +84,10 @@ CanonPath SourceAccessor::resolveSymlinks( todo.pop_front(); if (c == "" || c == ".") ; - else if (c == "..") - res.pop(); - else { + else if (c == "..") { + if (!res.isRoot()) + res.pop(); + } else { res.push(c); if (mode == SymlinkResolution::Full || !todo.empty()) { if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { From 57fea81f8a6ab3b91b1003d27bd48effad30b25a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Nov 2024 15:59:43 +0100 Subject: [PATCH 074/243] Work around gcc warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This shuts up a 300-line warning that includes /nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_tree.h:182:25: warning: ‘*(std::_Rb_tree_header*)((char*)& + offsetof(nix::value_type, nix::DerivedPath::.std::variant::.std::__detail::__variant::_Variant_base::.std::__detail::__variant::_Move_assign_base::.std::__detail::__variant::_Copy_assign_base::.std::__detail::__variant::_Move_ctor_base::.std::__detail::__variant::_Copy_ctor_base::.std::__detail::__variant::_Variant_storage::_M_u) + 24).std::_Rb_tree_header::_M_header.std::_Rb_tree_node_base::_M_parent’ may be used uninitialized [-Wmaybe-uninitialized] 182 | if (__x._M_header._M_parent != nullptr) | ~~~~~~~~~~~~~~^~~~~~~~~ --- src/nix/flake.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce2faacb0..925b66d3e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -643,10 +643,11 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { + auto path = DerivedPath::Built { .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); + }; + drvPaths.push_back(std::move(path)); } } } From fafaec5ac35d517a6e6217416336346b7353ea05 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:05:08 +0300 Subject: [PATCH 075/243] fix(treewide): remove unnecessary copying in range for loops This gets rid of unnecessary copies in range-based-for loops and local variables, when they are used solely as `const &`. Also added a fixme comment about a suspicious move out of const, which might not be intended. --- src/libcmd/installables.cc | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/context.cc | 2 ++ src/libexpr/primops/fetchClosure.cc | 2 +- src/libstore/build/drv-output-substitution-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/keys.cc | 4 ++-- src/libstore/local-overlay-store.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/store-api.cc | 4 ++-- src/libutil-tests/hilite.cc | 3 +-- src/libutil/args.cc | 2 +- src/libutil/url.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/flake.cc | 2 +- src/nix/hash.cc | 4 ++-- src/nix/run.cc | 4 ++-- src/nix/verify.cc | 2 +- 20 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f2b27af7c..250cd1413 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -858,7 +858,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() applyDefaultInstallables(rawInstallables); std::vector res; res.reserve(rawInstallables.size()); - for (auto i : rawInstallables) + for (const auto & i : rawInstallables) res.push_back(parseFlakeRefWithFragment( fetchSettings, expandTilde(i), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d2f75373..7e13e945c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4383,7 +4383,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) for (auto i = begin; i != end; ++i) { assert(idx <= 2 * len + 1 - 3); - auto match = *i; + const auto & match = *i; // Add a string for non-matched characters. list[idx++] = mkString(state, match.prefix()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 02683b173..ede7d97ba 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -132,6 +132,8 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V }, [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { /* Reuse original item because we want this to be idempotent. */ + /* FIXME: Suspicious move out of const. This is actually a copy, so the comment + above does not make much sense. */ return std::move(c); }, }, context.begin()->raw) }), diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index fc5bb3145..04b8d0595 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -40,7 +40,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor }); } - auto toPath = *toPathMaybe; + const auto & toPath = *toPathMaybe; // check and return diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index dedcad2b1..f069c0d94 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -32,7 +32,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); /* The callback of the curl download below can outlive `this` (if diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 315500719..983c86601 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -57,7 +57,7 @@ Goal::Co PathSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); cleanup(); diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc index 70478e7ad..668725fc7 100644 --- a/src/libstore/keys.cc +++ b/src/libstore/keys.cc @@ -10,12 +10,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.trustedPublicKeys.get()) { + for (const auto & s : settings.trustedPublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.secretKeyFiles.get()) { + for (const auto & secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index b86beba2c..56ff6bef3 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -156,7 +156,7 @@ void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); - for (auto p : lowerStore->queryValidDerivers(path)) + for (const auto & p : lowerStore->queryValidDerivers(path)) res.insert(p); return res; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8b2557060..27fcc2864 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -118,7 +118,7 @@ std::string NarInfo::to_string(const Store & store) const if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - for (auto sig : sigs) + for (const auto & sig : sigs) res += "Sig: " + sig + "\n"; if (ca) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bcbf0b55e..bf351a56d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -454,7 +454,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual debug("got %d keys, next marker '%s'", contents.size(), res.GetNextMarker()); - for (auto object : contents) { + for (const auto & object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 10577fa2a..78cc3b917 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1332,7 +1332,7 @@ ref openStore(StoreReference && storeURI) return std::make_shared(params); }, [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) + for (const auto & implem : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) return implem.create(g.scheme, g.authority, params); @@ -1363,7 +1363,7 @@ std::list> getDefaultSubstituters() } }; - for (auto uri : settings.substituters.get()) + for (const auto & uri : settings.substituters.get()) addStore(uri); stores.sort([](ref & a, ref & b) { diff --git a/src/libutil-tests/hilite.cc b/src/libutil-tests/hilite.cc index 1ff5980d5..5ef581888 100644 --- a/src/libutil-tests/hilite.cc +++ b/src/libutil-tests/hilite.cc @@ -52,8 +52,7 @@ namespace nix { std::regex("pt"), }; std::vector matches; - for(auto regex : regexes) - { + for (const auto & regex : regexes) { for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { matches.push_back(*it); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 385b6cd34..05ecf724e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -348,7 +348,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) /* Now that all the other args are processed, run the deferred completions. */ - for (auto d : deferredCompletions) + for (const auto & d : deferredCompletions) d.completer(*completions, d.n, d.prefix); } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9ed49dcbe..63b9734ee 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -77,7 +77,7 @@ std::map decodeQuery(const std::string & query) { std::map result; - for (auto s : tokenizeString(query, "&")) { + for (const auto & s : tokenizeString(query, "&")) { auto e = s.find('='); if (e == std::string::npos) { warn("dubious URI query '%s' is missing equal sign '%s', ignoring", s, "="); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c823c930e..b731b25af 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -222,7 +222,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { auto method = FileIngestionMethod::Flat; - for (auto i : opFlags) + for (const auto & i : opFlags) if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9a95bc695..1736add9a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -611,7 +611,7 @@ struct CmdDevelop : Common, MixEnvironment else if (!command.empty()) { std::vector args; args.reserve(command.size()); - for (auto s : command) + for (const auto & s : command) args.push_back(shellEscape(s)); script += fmt("exec %s\n", concatStringsSep(" ", args)); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 925b66d3e..cbc3cdb65 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -94,7 +94,7 @@ public: .label="inputs", .optional=true, .handler={[&](std::vector inputsToUpdate){ - for (auto inputToUpdate : inputsToUpdate) { + for (const auto & inputToUpdate : inputsToUpdate) { InputPath inputPath; try { inputPath = flake::parseInputPath(inputToUpdate); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 62266fda1..2f9b3fe7c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -79,7 +79,7 @@ struct CmdHashBase : Command void run() override { - for (auto path : paths) { + for (const auto & path : paths) { auto makeSink = [&]() -> std::unique_ptr { if (modulus) return std::make_unique(hashAlgo, *modulus); @@ -182,7 +182,7 @@ struct CmdToBase : Command void run() override { warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); - for (auto s : args) + for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; diff --git a/src/nix/run.cc b/src/nix/run.cc index c9857e13e..a9f9ef60f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -180,9 +180,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : fs::directory_iterator{"/"}) { + for (const auto & entry : fs::directory_iterator{"/"}) { checkInterrupt(); - auto src = entry.path(); + const auto & src = entry.path(); fs::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 124a05bed..52585fe08 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -129,7 +129,7 @@ struct CmdVerify : StorePathsCommand size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { - for (auto sig : sigs) { + for (const auto & sig : sigs) { if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig)) validSigs++; From f9980b5715fc403383f8e99d5da8bb91538c996d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Tue, 26 Nov 2024 00:13:54 +0300 Subject: [PATCH 076/243] fix(libutil/config): declare virtual dtor for AbstractConfig This prevents any potential cases of deletion through base pointer and its non-virtual dtor, which might leak memory. Also gets rid of the warning: /nix/store/fg7ass3a5m5pgl26qzfdniicbwbgzccy-gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:88:2: warning: destructor called on non-final 'nix::flake::Settings' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] 88 | __location->~_Tp(); .... ../src/libflake-c/nix_api_flake.cc:10:30: note: in instantiation of function template specialization 'nix::make_ref' requested here 10 | auto settings = nix::make_ref(); --- src/libutil/config.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index c0c59ac68..e98e09bf7 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -115,6 +115,8 @@ public: * Re-applies all previously attempted changes to unknown settings */ void reapplyUnknownSettings(); + + virtual ~AbstractConfig() = default; }; /** From 5b8728d393dd1c9bbbf6737500669853da7de1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 26 Nov 2024 07:07:31 +0100 Subject: [PATCH 077/243] more readable errors if symlinks cannot be created Before: filesystem error: cannot create symlink: Permission denied [/nix/store/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv] [/1s2p3a4rs172336hj2l8n20nz74hf71j-nix-eval-jobs-2.24.1.drv.tmp-2772352-1316231068] Now: creating symlink '/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv.tmp-2971297-324653080' -> '/nix/store/wfxz2q489c811n08cdqj7ywxm3n4z6m5-nix-eval-jobs-2.24.1.drv': Permission denied --- src/libutil/file-system.cc | 16 +++++++++++++--- src/libutil/file-system.hh | 2 -- src/nix/flake.cc | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 92996ea47..829700336 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -602,7 +602,11 @@ std::pair createTempFile(const Path & prefix) void createSymlink(const Path & target, const Path & link) { - fs::create_symlink(target, link); + try { + fs::create_symlink(target, link); + } catch (fs::filesystem_error & e) { + throw SysError("creating symlink '%1%' -> '%2%'", link, target); + } } void replaceSymlink(const fs::path & target, const fs::path & link) @@ -615,10 +619,16 @@ void replaceSymlink(const fs::path & target, const fs::path & link) fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw; + throw SysError("creating symlink '%1%' -> '%2%'", tmp, target); + } + + try { + fs::rename(tmp, link); + } catch (fs::filesystem_error & e) { + if (e.code() == std::errc::file_exists) continue; + throw SysError("renaming '%1%' to '%2%'", tmp, link); } - fs::rename(tmp, link); break; } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4c08cdf58..3c49181a0 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -250,8 +250,6 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st); /** * Create a symlink. * - * In the process of being deprecated for - * `std::filesystem::create_symlink`. */ void createSymlink(const Path & target, const Path & link); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 925b66d3e..4bb5c329e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -937,7 +937,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - fs::create_symlink(target, to2); + createSymlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); From d67aa03414ad6e75b2ac2145406fcc936c0f1798 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 26 Nov 2024 18:35:18 +0000 Subject: [PATCH 078/243] src/perl/meson.build: fall back to 'bz2' library lookup Upstream `bzip2` does not provide `pkg-config` files. As a result an attempt to build `nix` on some distributions like Gentoo failos the configure as: $ meson setup .. ... Executing subproject perl ... perl| Run-time dependency bzip2 found: NO (tried pkgconfig and cmake) ../src/perl/meson.build:68:12: ERROR: Dependency "bzip2" not found, tried pkgconfig and cmake The change falls back to `bz2` library for such cases. --- src/perl/meson.build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/perl/meson.build b/src/perl/meson.build index dcb6a68a4..52d85fd60 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -65,7 +65,13 @@ yath = find_program('yath', required : false) # Required Libraries #------------------------------------------------- -bzip2_dep = dependency('bzip2') +bzip2_dep = dependency('bzip2', required: false) +if not bzip2_dep.found() + bzip2_dep = cpp.find_library('bz2') + if not bzip2_dep.found() + error('No "bzip2" pkg-config or "bz2" library found') + endif +endif curl_dep = dependency('libcurl') libsodium_dep = dependency('libsodium') From 2679e55232af74b0325877b6a49ed83502711fc0 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 26 Nov 2024 23:08:10 +0000 Subject: [PATCH 079/243] tests/functional/meson.build: always look up `ls` as a `coreutils` proxy Without the change `meson setup` fails on `Gentoo or Debian as those don't use multicall binary: $ meson setup .. ... Executing subproject nix-functional-tests ... ../src/nix-functional-tests/meson.build:24:14: ERROR: Program 'coreutils' not found or not executable The change always uses `ls` to look `coreutils` up. Closes: https://github.com/NixOS/nix/issues/11975 --- tests/functional/meson.build | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 0d46f9ce2..933595cd5 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -17,12 +17,10 @@ fs = import('fs') nix = find_program('nix') bash = find_program('bash', native : true) busybox = find_program('busybox', native : true, required : false) -if host_machine.system() == 'windows' - # Because of the state of symlinks on Windows, coreutils.exe doesn't usually exist, but things like ls.exe will - coreutils = find_program('ls', native : true) -else - coreutils = find_program('coreutils', native : true) -endif +# Look up `coreutils` package by searching for `ls` binary. +# Previously we looked up `coreutils` on `linux`, but that is not +# guaranteed to exist either. +coreutils = find_program('ls', native : true) dot = find_program('dot', native : true, required : false) nix_bin_dir = fs.parent(nix.full_path()) From 21ddd2022e40d4727684cadf7aca44b0b4ec622c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 27 Nov 2024 07:39:30 +0100 Subject: [PATCH 080/243] mergify: drop installer test --- .mergify.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index c297d3d5e..ac1bee111 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,9 +2,6 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=installer - - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) - check-success=vm_tests From a5c7709f97ceb567ffd1903aa1cd921ca70d7c7b Mon Sep 17 00:00:00 2001 From: h0nIg Date: Wed, 27 Nov 2024 13:24:46 +0100 Subject: [PATCH 081/243] docker: Fix command "nix profile install", Don't require --impure --- docker.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index 19479e57c..e2e9da728 100644 --- a/docker.nix +++ b/docker.nix @@ -258,14 +258,14 @@ let mkdir -p $out/nix/var/nix/profiles/per-user/${uname} ln -s ${profile} $out/nix/var/nix/profiles/default-1-link - ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels mkdir -p $out${userHome}/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels mkdir -p $out/bin $out/usr/bin From 37fd80588fff0ee5e9e27c9d6a1dbc7d2f740b6d Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 27 Nov 2024 17:11:36 +0300 Subject: [PATCH 082/243] shellcheck: simplify install-nix-from-tarball.sh --- maintainers/flake-module.nix | 1 - scripts/install-nix-from-tarball.sh | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index ba6cd2816..1d4e85c8c 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -496,7 +496,6 @@ ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-nix-from-tarball\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 794622530..007fe85ee 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -48,15 +48,14 @@ case "$(uname -s)" in INSTALL_MODE=no-daemon;; esac -# space-separated string -ACTIONS= +ACTION= # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) INSTALL_MODE=daemon - ACTIONS="${ACTIONS}install " + ACTION=install ;; --no-daemon) if [ "$(uname -s)" = "Darwin" ]; then @@ -65,18 +64,14 @@ while [ $# -gt 0 ]; do fi INSTALL_MODE=no-daemon # intentional tail space - ACTIONS="${ACTIONS}install " + ACTION=install ;; - # --uninstall) - # # intentional tail space - # ACTIONS="${ACTIONS}uninstall " - # ;; --yes) export NIX_INSTALLER_YES=1;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) - export NIX_USER_COUNT=$2 + export NIX_USER_COUNT="$2" shift;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; @@ -128,7 +123,7 @@ done if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" $ACTIONS # let ACTIONS split + exec "$self/install-multi-user" $ACTION exit 0 fi From 985b2f9df30e62a21c32d7ffaae7aebfe4c56d3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2024 15:23:56 +0100 Subject: [PATCH 083/243] Remove FIXME --- src/libflake/flake/flake.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 8e381c0d2..9d5760ed1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -638,10 +638,6 @@ LockedFlake lockFlake( nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - // FIXME: sourcePath is wrong here, we - // should pass a lambda that lazily - // fetches the parent flake if needed - // (i.e. getInputFlake()). computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } From 8034589d7eb299c126f169ff0780b5242936acdd Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 14 Aug 2024 00:05:06 -0400 Subject: [PATCH 084/243] parser-state: fix attribute merging --- src/libexpr-tests/trivial.cc | 51 +++++++++ src/libexpr/parser-state.hh | 108 +++++++++++------- ...fail-attrset-merge-drops-later-rec.err.exp | 5 + ...val-fail-attrset-merge-drops-later-rec.nix | 1 + ...val-okay-regrettable-rec-attrset-merge.exp | 1 + ...val-okay-regrettable-rec-attrset-merge.nix | 3 + .../parse-fail-mixed-nested-attrs1.err.exp | 8 +- .../parse-fail-mixed-nested-attrs2.err.exp | 8 +- 8 files changed, 133 insertions(+), 52 deletions(-) create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp create mode 100644 tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp create mode 100644 tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix diff --git a/src/libexpr-tests/trivial.cc b/src/libexpr-tests/trivial.cc index e455a571b..d77b4d53b 100644 --- a/src/libexpr-tests/trivial.cc +++ b/src/libexpr-tests/trivial.cc @@ -177,6 +177,57 @@ namespace nix { ) ); +// The following macros ultimately define 48 tests (16 variations on three +// templates). Each template tests an expression that can be written in 2^4 +// different ways, by making four choices about whether to write a particular +// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };` +// (expanded). +// +// The nestedAttrsetMergeXXXX tests check that the expression +// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is +// expanded. (That exact expression is exercised in test +// nestedAttrsetMerge0000, because it is fully collapsed. The test +// nestedAttrsetMerge1001 would instead examine +// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.) +// +// The nestedAttrsetMergeDupXXXX tests check that the expression +// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again +// regardless of how it is expanded. +// +// The nestedAttrsetMergeLetXXXX tests check that the expression +// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is +// expanded. +#define X_EXPAND_IF0(k, v) k "." v +#define X_EXPAND_IF1(k, v) k " = { " v " };" +#define X4(w, x, y, z) \ + TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \ + auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \ + ASSERT_THAT(v, IsTrue()); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \ + ASSERT_THROW(eval("{ " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \ + auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \ + ASSERT_THAT(v, IsTrue()); \ + }; +#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1) +#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1) +#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1) + X1(0) X1(1) +#undef X_EXPAND_IF0 +#undef X_EXPAND_IF1 +#undef X1 +#undef X2 +#undef X3 +#undef X4 + TEST_F(TrivialExpressionTest, functor) { auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5"); ASSERT_THAT(v, IsIntEq(15)); diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 8ad0d9ad7..21a880e8e 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -88,6 +88,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); + void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + ExprAttrs * nested; if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); - attrs = attrs2; - } else + nested = dynamic_cast(j->second.e); + if (!nested) { + attrPath.erase(i + 1, attrPath.end()); dupAttr(attrPath, pos, j->second.pos); + } } else { - ExprAttrs * nested = new ExprAttrs; + nested = new ExprAttrs; attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; } } else { - ExprAttrs *nested = new ExprAttrs; + nested = new ExprAttrs; attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; } + attrs = nested; } // Expr insertion. // ========================== if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) - jAttrs->inheritFromExprs = std::make_unique>(); - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { - auto & sel = dynamic_cast(*ad.second.e); - auto & from = dynamic_cast(*sel.e); - from.displ += jAttrs->inheritFromExprs->size(); - } - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - if (ae->inheritFromExprs) { - jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), - ae->inheritFromExprs->begin(), ae->inheritFromExprs->end()); - } - } else { - dupAttr(attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } + addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } @@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const } } +/** + * Precondition: attrPath is used for error messages and should already contain + * symbol as its last element. + */ +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) +{ + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(def.e); + auto jAttrs = dynamic_cast(j->second.e); + + // N.B. In a world in which we are less bound by our past mistakes, we + // would also test that jAttrs and ae are not recursive. The effect of + // not doing so is that any `rec` marker on ae is discarded, and any + // `rec` marker on jAttrs will apply to the attributes in ae. + // See https://github.com/NixOS/nix/issues/9020. + if (jAttrs && ae) { + if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) + jAttrs->inheritFromExprs = std::make_unique>(); + for (auto & ad : ae->attrs) { + if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { + auto & sel = dynamic_cast(*ad.second.e); + auto & from = dynamic_cast(*sel.e); + from.displ += jAttrs->inheritFromExprs->size(); + } + attrPath.emplace_back(AttrName(ad.first)); + addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); + attrPath.pop_back(); + } + ae->attrs.clear(); + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), + std::make_move_iterator(ae->dynamicAttrs.begin()), + std::make_move_iterator(ae->dynamicAttrs.end())); + ae->dynamicAttrs.clear(); + if (ae->inheritFromExprs) { + jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), + std::make_move_iterator(ae->inheritFromExprs->begin()), + std::make_move_iterator(ae->inheritFromExprs->end())); + ae->inheritFromExprs = nullptr; + } + } else { + dupAttr(attrPath, def.pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(symbol, def); + def.e->setName(symbol); + } +} + inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) { std::sort(formals->formals.begin(), formals->formals.end(), diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp new file mode 100644 index 000000000..d1cdc7b76 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp @@ -0,0 +1,5 @@ +error: undefined variable 'd' + at /pwd/lang/eval-fail-attrset-merge-drops-later-rec.nix:1:26: + 1| { a.b = 1; a = rec { c = d + 2; d = 3; }; }.c + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix new file mode 100644 index 000000000..fdb314b91 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix @@ -0,0 +1 @@ +{ a.b = 1; a = rec { c = d + 2; d = 3; }; }.c diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp new file mode 100644 index 000000000..1e8b31496 --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp @@ -0,0 +1 @@ +6 diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix new file mode 100644 index 000000000..8df6a2ad8 --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix @@ -0,0 +1,3 @@ +# This is for backwards compatibility, not because we like it. +# See https://github.com/NixOS/nix/issues/9020. +{ a = rec { b = c + 1; d = 2; }; a.c = d + 3; }.a.b diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp index a4472156b..49a07323f 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -1,6 +1,6 @@ -error: attribute 'z' already defined at «stdin»:3:16 - at «stdin»:2:3: - 1| { +error: attribute 'x.z' already defined at «stdin»:2:3 + at «stdin»:3:16: 2| x.z = 3; - | ^ 3| x = { y = 3; z = 3; }; + | ^ + 4| } diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp index ead1f0dbd..36fab2fe6 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -1,6 +1,6 @@ -error: attribute 'y' already defined at «stdin»:3:9 - at «stdin»:2:3: - 1| { +error: attribute 'x.y.y' already defined at «stdin»:2:3 + at «stdin»:3:9: 2| x.y.y = 3; - | ^ 3| x = { y.y= 3; z = 3; }; + | ^ + 4| } From e5e09006f97700f35c68411df6fe4f8a9d7dc807 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Nov 2024 15:25:51 +0100 Subject: [PATCH 085/243] Work around gcc warning Same as 57fea81f8a6ab3b91b1003d27bd48effad30b25a. --- src/nix-env/nix-env.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ba2baccee..e9eb52708 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,12 +481,13 @@ static void printMissing(EvalState & state, PackageInfos & elems) { std::vector targets; for (auto & i : elems) - if (auto drvPath = i.queryDrvPath()) - targets.emplace_back(DerivedPath::Built{ + if (auto drvPath = i.queryDrvPath()) { + auto path = DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); - else + }; + targets.emplace_back(std::move(path)); + } else targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); From 04975f7c32e91cda35adc0c3257c98542be97d0e Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Mon, 28 Oct 2024 09:53:44 +0300 Subject: [PATCH 086/243] install: Allow to specify alternative `sudo` command --- scripts/install-multi-user.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a487d459f..518c955b4 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -56,6 +56,9 @@ readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" readonly EXTRACTED_NIX_PATH="$(dirname "$0")" +# allow to override identity change command +readonly NIX_BECOME=${NIX_BECOME:-sudo} + readonly ROOT_HOME=~root if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then @@ -123,7 +126,7 @@ uninstall_directions() { cat < Date: Sat, 23 Nov 2024 09:51:56 +0300 Subject: [PATCH 087/243] Allow `sudo` alternatives when installing from tarball --- scripts/install-nix-from-tarball.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 007fe85ee..8d127a9c5 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -9,6 +9,8 @@ self="$(dirname "$0")" nix="@nix@" cacert="@cacert@" +# allow to override identity change command +readonly NIX_BECOME="${NIX_BECOME:-sudo}" if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 @@ -63,7 +65,6 @@ while [ $# -gt 0 ]; do exit 1 fi INSTALL_MODE=no-daemon - # intentional tail space ACTION=install ;; --yes) @@ -135,8 +136,8 @@ echo "performing a single-user installation of Nix..." >&2 if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" - echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 - if ! sudo sh -c "$cmd"; then + echo "directory $dest does not exist; creating it by running '$cmd' using $NIX_BECOME" >&2 + if ! $NIX_BECOME sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 exit 1 fi From 747cf4e50f43b510ff054ec14bdef87634231237 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Nov 2024 22:49:04 +0100 Subject: [PATCH 088/243] fix: Add splicing to fix the manual in cross We *could* use a "native" manual instead - ie reusing a native `nixpkgsFor.${buildPlatform}`, but this works, and also works for possible cases where we have a custom or patched build tool. --- flake.nix | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 06025e3b7..794736af4 100644 --- a/flake.nix +++ b/flake.nix @@ -124,18 +124,36 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { - inherit (final) lib; - inherit officialRelease; - src = self; - }); + nixComponents = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final.nixDependencies) newScope; + } + { + otherSplices = final.generateSplicesForMkScope "nixComponents"; + f = import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }; + }; # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. - nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv; - pkgs = final; - }); + nixDependencies = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above + } + { + otherSplices = final.generateSplicesForMkScope "nixDependencies"; + f = import ./packaging/dependencies.nix { + inherit inputs stdenv; + pkgs = final; + }; + }; nix = final.nixComponents.nix-cli; From d67e24afec185eee740180b5f023fc4df462013c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Dec 2024 17:42:11 +0100 Subject: [PATCH 089/243] fix: Add missing manpages to meson.build --- doc/manual/meson.build | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 3630e2dc8..c4cc9b893 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -199,6 +199,7 @@ nix3_manpages = [ 'nix3-build', 'nix3-bundle', 'nix3-config', + 'nix3-config-check', 'nix3-config-show', 'nix3-copy', 'nix3-daemon', @@ -206,8 +207,8 @@ nix3_manpages = [ 'nix3-derivation', 'nix3-derivation-show', 'nix3-develop', - #'nix3-doctor', 'nix3-edit', + 'nix3-env-shell', 'nix3-eval', 'nix3-flake-archive', 'nix3-flake-check', @@ -224,6 +225,7 @@ nix3_manpages = [ 'nix3-fmt', 'nix3-hash-file', 'nix3-hash', + 'nix3-hash-convert', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -238,6 +240,7 @@ nix3_manpages = [ 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', + 'nix3-nar-pack', 'nix3-nar', 'nix3-path-info', 'nix3-print-dev-env', @@ -260,7 +263,7 @@ nix3_manpages = [ 'nix3-repl', 'nix3-run', 'nix3-search', - #'nix3-shell', + 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', 'nix3-store-cat', @@ -270,6 +273,7 @@ nix3_manpages = [ 'nix3-store-diff-closures', 'nix3-store-dump-path', 'nix3-store-gc', + 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', 'nix3-store', From 038ab46d7a08b34e9706a5f7a3c75995cd145899 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Dec 2024 16:37:07 +0100 Subject: [PATCH 090/243] Restore org.nixos.nix-daemon.plist installation --- misc/launchd/meson.build | 13 +++++++++++++ misc/meson.build | 1 + 2 files changed, 14 insertions(+) create mode 100644 misc/launchd/meson.build diff --git a/misc/launchd/meson.build b/misc/launchd/meson.build new file mode 100644 index 000000000..5168131d1 --- /dev/null +++ b/misc/launchd/meson.build @@ -0,0 +1,13 @@ +configure_file( + input : 'org.nixos.nix-daemon.plist.in', + output : 'org.nixos.nix-daemon.plist', + install : true, + install_dir : get_option('prefix') / 'Library/LaunchDaemons', + install_mode : 'rw-r--r--', + configuration : { + # TODO: unhardcode paths with something like: + # 'storedir' : store_dir, + # 'localstatedir' : localstatedir, + # 'bindir' : bindir, + }, +) diff --git a/misc/meson.build b/misc/meson.build index a6d1f944b..78ae3c06c 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,4 +2,5 @@ subdir('bash') subdir('fish') subdir('zsh') +subdir('launchd') subdir('systemd') From 63c0f0dcd258b6763970a80c43bed431e691e5ba Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Dec 2024 16:51:01 +0100 Subject: [PATCH 091/243] Install init system configs only when relevant --- misc/meson.build | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/misc/meson.build b/misc/meson.build index 78ae3c06c..82f2b0c65 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,5 +2,10 @@ subdir('bash') subdir('fish') subdir('zsh') -subdir('launchd') -subdir('systemd') +if host_machine.system() == 'linux' + subdir('systemd') +endif + +if host_machine.system() == 'darwin' + subdir('launchd') +endif From abcfdb4bdab966016d3c9d725fafdad8eadbbf5d Mon Sep 17 00:00:00 2001 From: mupdt <25388474+mupdt@users.noreply.github.com> Date: Wed, 4 Dec 2024 05:49:21 -0500 Subject: [PATCH 092/243] s3-binary-cache: show the error's request ID The request ID is essential for traceability and debugging purposes. It allows us to connect client-side to server-side events. --- src/libstore/s3-binary-cache-store.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bf351a56d..cfa713b00 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -48,7 +48,11 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - s + ": " + outcome.GetError().GetMessage()); + fmt( + "%s: %s (request id: %s)", + s, + outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId())); return outcome.GetResultWithOwnership(); } @@ -121,9 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) - printError("AWS error '%s' (%s), will retry in %d ms", + printError("AWS error '%s' (%s; request id: %s), will retry in %d ms", error.GetExceptionName(), error.GetMessage(), + error.GetRequestId(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } From 3b21ea40cc6b6b9d512c15facb2e24c32a368993 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 16:52:30 +0100 Subject: [PATCH 093/243] HttpBinaryCacheStore: Improve error message for unauthorized caches Instead of the unhelpful warning: 'https://cache.flakehub.com' does not appear to be a binary cache you now get warning: unable to download 'https://cache.flakehub.com/nix-cache-info': HTTP error 401 response body: {"code":401,"error":"Unauthorized","message":"Unauthorized."} --- src/libstore/binary-cache-store.cc | 13 ++++++++----- src/libstore/binary-cache-store.hh | 8 ++++++++ src/libstore/http-binary-cache-store.cc | 13 +++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b3..896779f85 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -39,15 +39,13 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) void BinaryCacheStore::init() { - std::string cacheInfoFile = "nix-cache-info"; - - auto cacheInfo = getFile(cacheInfoFile); + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon= line.find(':'); - if (colon ==std::string::npos) continue; + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { @@ -63,6 +61,11 @@ void BinaryCacheStore::init() } } +std::optional BinaryCacheStore::getNixCacheInfo() +{ + return getFile(cacheInfoFile); +} + void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc9252..6bd7fd14a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -64,6 +64,8 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; + const std::string cacheInfoFile = "nix-cache-info"; + BinaryCacheStore(const Params & params); public: @@ -84,6 +86,12 @@ public: */ virtual void getFile(const std::string & path, Sink & sink); + /** + * Get the contents of /nix-cache-info. Return std::nullopt if it + * doesn't exist. + */ + virtual std::optional getNixCacheInfo(); + /** * Fetch the specified file and call the specified callback with * the result. A subclass may implement this asynchronously. diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index fc7ac2dea..f32616f94 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,19 @@ protected: } } + std::optional getNixCacheInfo() override + { + try { + auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + return result.data; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) + return std::nullopt; + maybeDisable(); + throw; + } + } + /** * This isn't actually necessary read only. We support "upsert" now, so we * have a notion of authentication via HTTP POST/PUT. From d1894f34566cb1a7c56b4225e0faf700dfd5b22d Mon Sep 17 00:00:00 2001 From: wh0 Date: Wed, 4 Dec 2024 23:21:07 -0800 Subject: [PATCH 094/243] tests: derivation-advanced-attributes unset NIX_STORE when built by nix, NIX_STORE is set, which breaks $got when it is not the default /nix/store --- tests/functional/derivation-advanced-attributes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 271f17dc6..6707b345c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -13,7 +13,7 @@ badExitCode=0 store="$TEST_ROOT/store" for nixFile in derivation/*.nix; do - drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" expected="derivation/$testName.drv" From 52f1cd05956b7745985c5565a463633f0b805639 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:02:35 +0100 Subject: [PATCH 095/243] nix hash convert: Support SRI hashes that lack trailing '=' characters Fixes #11996. --- src/libutil/hash.cc | 3 ++- src/nix/hash.cc | 9 ++++++--- tests/functional/hash-convert.sh | 14 ++++++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 748176d33..b69dec685 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -134,7 +134,8 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const Hash Hash::dummy(HashAlgorithm::SHA256); -Hash Hash::parseSRI(std::string_view original) { +Hash Hash::parseSRI(std::string_view original) +{ auto rest = original; // Parse the has type before the separater, if there was one. diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2f9b3fe7c..654e67437 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -222,9 +222,12 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, algo); - if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + for (const auto & s: hashStrings) { + Hash h = + from == HashFormat::SRI + ? Hash::parseSRI(s) + : Hash::parseAny(s, algo); + if (from && from != HashFormat::SRI && h.to_string(*from, false) != s) { auto from_as_string = printHashFormat(*from); throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); } diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 3a099950f..1cbe864f3 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -93,15 +93,17 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] + expectStderr 1 nix hash convert --hash-algo "$1" --from sri "$2" | grepQuiet "is not SRI" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$2" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility. +[[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] +[[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] From 33b645cedfabd4f275644ac4df0eac220f444278 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:19:21 +0100 Subject: [PATCH 096/243] nix hash convert: Don't fail on uppercase base-16 hashes --- src/nix/hash.cc | 10 +++++++--- tests/functional/hash-convert.sh | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 654e67437..fd07fa7d1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -222,14 +222,18 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto & s: hashStrings) { + for (const auto & s : hashStrings) { Hash h = from == HashFormat::SRI ? Hash::parseSRI(s) : Hash::parseAny(s, algo); - if (from && from != HashFormat::SRI && h.to_string(*from, false) != s) { + if (from + && from != HashFormat::SRI + && h.to_string(*from, false) != + (from == HashFormat::Base16 ? toLower(s) : s)) + { auto from_as_string = printHashFormat(*from); - throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + throw BadHash("input hash '%s' does not have the expected format for '--from %s'", s, from_as_string); } logger->cout(h.to_string(to, to == HashFormat::SRI)); } diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 1cbe864f3..c40cb469c 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -98,6 +98,8 @@ try3() { expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" + # Base-16 hashes can be in uppercase. + nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" From 408c2faf9365ab1b6f9e28551429d9352f9e7f1a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Dec 2024 16:25:05 +0100 Subject: [PATCH 097/243] nix hash: Don't print 'nix hash' deprecation message Fixes #11997. --- src/nix/hash.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index fd07fa7d1..416cd19b3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -163,8 +163,11 @@ struct CmdToBase : Command HashFormat hashFormat; std::optional hashAlgo; std::vector args; + bool legacyCli; - CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) + CmdToBase(HashFormat hashFormat, bool legacyCli = false) + : hashFormat(hashFormat) + , legacyCli(legacyCli) { addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); @@ -181,7 +184,8 @@ struct CmdToBase : Command void run() override { - warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); + if (!legacyCli) + warn("The old format conversion subcommands of `nix hash` were deprecated in favor of `nix hash convert`."); for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -328,7 +332,7 @@ static int compatNixHash(int argc, char * * argv) } else { - CmdToBase cmd(hashFormat); + CmdToBase cmd(hashFormat, true); cmd.args = ss; if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); From 4137ead7a1011c3f410899be089822e8cf33862e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 5 Dec 2024 14:01:00 -0500 Subject: [PATCH 098/243] Disable suid and atime on the /nix mount point on Darwin The Determinate Nix Installer has set nosuid and noatime in https://github.com/DeterminateSystems/nix-installer/pull/1338, and figured this perf and security improvement is worthy of upstreaming. The /nix volume shouldn't have setuid binaries anyway, and filesystems seem to generally be noatime on macOS. Further, the garbage collector doesn't use atime. --- scripts/create-darwin-volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 103e1e391..7a61764d4 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -463,7 +463,7 @@ EOF EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" < Date: Thu, 5 Dec 2024 20:10:59 +0100 Subject: [PATCH 099/243] Fix typo (#12015) --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7e13e945c..5202ef7d7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1100,7 +1100,7 @@ static RegisterPrimOp primop_warn({ .name = "__warn", .args = {"e1", "e2"}, .doc = R"( - Evaluate *e1*, which must be a string and print iton standard error as a warning. + Evaluate *e1*, which must be a string, and print it on standard error as a warning. Then return *e2*. This function is useful for non-critical situations where attention is advisable. From ad296eae2a1f1b6ad6399f364f8dc91c80923922 Mon Sep 17 00:00:00 2001 From: "Shahar \"Dawn\" Or" Date: Sat, 7 Dec 2024 06:37:30 +0700 Subject: [PATCH 100/243] Test: more specific error message for `head` Sorry, I'm not sure how to implement this. So just a test change. And hopefully will be picked up by someone who is paying attention. A hero. --- src/libexpr-tests/error_traces.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index be379a909..011a0848d 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -712,7 +712,7 @@ namespace nix { ASSERT_TRACE1("head []", Error, - HintFmt("list index %d is out of bounds", 0)); + HintFmt("'head' called on an empty list")); } From ad3a67a2a0dc815219593a7aae306db8ed8ec3a3 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 7 Dec 2024 04:42:04 +0100 Subject: [PATCH 101/243] optimize string concat --- src/libutil/util.hh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0fb6ff837..4d5683e2b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -338,7 +338,9 @@ std::string showBytes(uint64_t bytes); */ inline std::string operator + (const std::string & s1, std::string_view s2) { - auto s = s1; + std::string s; + s.reserve(s1.size() + s2.size()); + s.append(s1); s.append(s2); return s; } @@ -351,10 +353,11 @@ inline std::string operator + (std::string && s, std::string_view s2) inline std::string operator + (std::string_view s1, const char * s2) { + auto s2Size = strlen(s2); std::string s; - s.reserve(s1.size() + strlen(s2)); + s.reserve(s1.size() + s2Size); s.append(s1); - s.append(s2); + s.append(s2, s2Size); return s; } From 8b9e0f86e406b53911b6c2d1ec4066a767c44b3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Dec 2024 13:56:03 +0100 Subject: [PATCH 102/243] nix flake lock: Fail if there is an unlocked input Since the only purpose of `nix flake lock` is to write a new lock file, it should be a fatal error if we can't write the lock file. --- src/libflake/flake/flake.cc | 2 ++ src/libflake/flake/flake.hh | 5 +++++ src/nix/flake.cc | 1 + tests/functional/flakes/unlocked-override.sh | 3 +++ 4 files changed, 11 insertions(+) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 19b622a34..2165ffd8d 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -672,6 +672,8 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { + if (lockFlags.failOnUnlocked) + throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index cc2bea76e..0dfd9440d 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -156,6 +156,11 @@ struct LockFlags */ bool writeLockFile = true; + /** + * Throw an exception when the flake has an unlocked input. + */ + bool failOnUnlocked = false; + /** * Whether to use the registries to lookup indirect flake * references like 'nixpkgs'. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9f3584a11..55aa8971e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -162,6 +162,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; lockFlake(); diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index a17a0c2af..ebad332d0 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -30,3 +30,6 @@ git -C "$flake2Dir" add flake.nix echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] + +expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | + grepQuiet "cannot write lock file.*because it has an unlocked input" From e82ff51726b33b95152c4c64dc870eab831936f9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 20 Nov 2024 11:05:55 +0100 Subject: [PATCH 103/243] tests/functional/dependencies.nix: Refactor, replace arcane let --- tests/functional/dependencies.nix | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a..4ff29227f 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,7 +1,7 @@ { hashInvalidator ? "" }: with import ./config.nix; -let { +let input0 = mkDerivation { name = "dependencies-input-0"; @@ -33,16 +33,15 @@ let { outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - body = mkDerivation { - name = "dependencies-top"; - builder = ./dependencies.builder0.sh + "/FOOBAR/../."; - input1 = input1 + "/."; - input2 = "${input2}/."; - input1_drv = input1; - input2_drv = input2; - input0_drv = input0; - fod_input_drv = fod_input; - meta.description = "Random test package"; - }; - +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + fod_input_drv = fod_input; + meta.description = "Random test package"; } From ee03fd478e99d90eb02baefda6dee7eb9a3c347f Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Fri, 11 Oct 2024 17:16:32 +0200 Subject: [PATCH 104/243] libutil: handle json builder log messages with unexpected format Before this change, expressions like: with import {}; runCommand "foo" {} '' echo '@nix {}' >&$NIX_LOG_FD '' would result in Lix crashing, because accessing nonexistent fields of a JSON object throws an exception. Rather than handling each field individually, we just catch JSON exceptions wholesale. Since these log messages are an unusual circumstance, log a warning when this happens. Fixes #544. Change-Id: Idc2d8acf6e37046b3ec212f42e29269163dca893 (cherry picked from commit e55cd3beea710db727fd966f265a1b715b7285f3) --- src/libutil/logging.cc | 66 +++++++++++++++++-------------- tests/functional/dependencies.nix | 13 ++++++ 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 80c107ef5..0eceee6d4 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -295,37 +295,45 @@ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, bool trusted) { - std::string action = json["action"]; + try { + std::string action = json["action"]; - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + return true; + } catch (const nlohmann::json::exception &e) { + warn( + "warning: Unable to handle a JSON message from the builder: %s", + e.what() + ); + return false; } - - else if (action == "stop") - activities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } - - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } - - return true; } bool handleJSONLogMessage(const std::string & msg, diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index 4ff29227f..64d9d2360 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -33,6 +33,18 @@ let outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; + unusual-logging = mkDerivation { + name = "unusual-logging"; + buildCommand = '' + { + echo "@nix 1" + echo "@nix {}" + echo '@nix {"action": null}' + } >&$NIX_LOG_FD + touch $out + ''; + }; + in mkDerivation { name = "dependencies-top"; @@ -42,6 +54,7 @@ mkDerivation { input1_drv = input1; input2_drv = input2; input0_drv = input0; + unusual_logging_drv = unusual-logging; fod_input_drv = fod_input; meta.description = "Random test package"; } From 1485937b897bc88cda0848b54b8ad9310230c47b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 20 Nov 2024 11:16:39 +0100 Subject: [PATCH 105/243] tests/functional/dependencies.nix: Check that we tolerate syntax and type errors --- tests/functional/dependencies.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index 64d9d2360..d177f5b63 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -40,6 +40,8 @@ let echo "@nix 1" echo "@nix {}" echo '@nix {"action": null}' + echo '@nix {"action": 123}' + echo '@nix ][' } >&$NIX_LOG_FD touch $out ''; From 03d4bfd852dce9a050f984e887c887a43581796c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 21 Nov 2024 16:03:21 +0100 Subject: [PATCH 106/243] Push log source description out of libutil and report build hook @nix warning correctly --- src/libstore/build/derivation-goal.cc | 8 ++++---- src/libutil/logging.cc | 17 ++++++++++------- src/libutil/logging.hh | 13 ++++++++++++- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf1a25db1..2ff0ef92f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,7 +1161,7 @@ HookReply DerivationGoal::tryBuildHook() throw; } }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true)) ; else if (s.substr(0, 2) == "# ") { reply = s.substr(2); @@ -1346,9 +1346,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - auto json = parseJSONMessage(currentHookLine); + auto json = parseJSONMessage(currentHookLine, "the derivation builder"); if (json) { - auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. if (s && !isWrittenToLog && logSink) { @@ -1390,7 +1390,7 @@ void DerivationGoal::handleEOF(Descriptor fd) void DerivationGoal::flushLine() { - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) ; else { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 0eceee6d4..4c7061016 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -280,20 +280,22 @@ static Logger::Fields getFields(nlohmann::json & json) return fields; } -std::optional parseJSONMessage(const std::string & msg) +std::optional parseJSONMessage(const std::string & msg, std::string_view source) { if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { return nlohmann::json::parse(std::string(msg, 5)); } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + printError("bad JSON log message from %s: %s", + Uncolored(source), + e.what()); } return std::nullopt; } bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, - bool trusted) + std::string_view source, bool trusted) { try { std::string action = json["action"]; @@ -329,7 +331,8 @@ bool handleJSONLogMessage(nlohmann::json & json, return true; } catch (const nlohmann::json::exception &e) { warn( - "warning: Unable to handle a JSON message from the builder: %s", + "warning: Unable to handle a JSON message from %s: %s", + Uncolored(source), e.what() ); return false; @@ -337,12 +340,12 @@ bool handleJSONLogMessage(nlohmann::json & json, } bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map & activities, bool trusted) + const Activity & act, std::map & activities, std::string_view source, bool trusted) { - auto json = parseJSONMessage(msg); + auto json = parseJSONMessage(msg, source); if (!json) return false; - return handleJSONLogMessage(*json, act, activities, trusted); + return handleJSONLogMessage(*json, act, activities, source, trusted); } Activity::~Activity() diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 250f92099..11e4033a5 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -185,14 +185,25 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); -std::optional parseJSONMessage(const std::string & msg); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ +std::optional parseJSONMessage(const std::string & msg, std::string_view source); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, + std::string_view source, bool trusted); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, + std::string_view source, bool trusted); /** From 8490fba42d49cec068eea0a442bdc125c0030cd2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Dec 2024 16:15:23 +0100 Subject: [PATCH 107/243] Improve error messages for head/elemAt --- src/libexpr-tests/error_traces.cc | 14 ++++++------ src/libexpr/primops.cc | 36 ++++++++++++++++--------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index 011a0848d..2aa13cf62 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -691,15 +691,15 @@ namespace nix { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); ASSERT_TRACE1("elemAt [] (-1)", Error, - HintFmt("list index %d is out of bounds", -1)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - HintFmt("list index %d is out of bounds", 3)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); } @@ -708,11 +708,11 @@ namespace nix { ASSERT_TRACE2("head 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.head'")); ASSERT_TRACE1("head []", Error, - HintFmt("'head' called on an empty list")); + HintFmt("'builtins.head' called on an empty list")); } @@ -721,11 +721,11 @@ namespace nix { ASSERT_TRACE2("tail 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("while evaluating the first argument passed to 'builtins.tail'")); ASSERT_TRACE1("tail []", Error, - HintFmt("'tail' called on an empty list")); + HintFmt("'builtins.tail' called on an empty list")); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5202ef7d7..7c5c6ea9b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3259,23 +3259,19 @@ static RegisterPrimOp primop_isList({ .fun = prim_isList, }); -static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) -{ - state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); - if (n < 0 || (unsigned int) n >= list.listSize()) - state.error( - "list index %1% is out of bounds", - n - ).atPos(pos).debugThrow(); - state.forceValue(*list.listElems()[n], pos); - v = *list.listElems()[n]; -} - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; - elemAt(state, pos, *args[0], elem, v); + NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value; + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'"); + if (n < 0 || (unsigned int) n >= args[0]->listSize()) + state.error( + "'builtins.elemAt' called with index %d on a list of size %d", + n, + args[0]->listSize() + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[n], pos); + v = *args[0]->listElems()[n]; } static RegisterPrimOp primop_elemAt({ @@ -3291,7 +3287,13 @@ static RegisterPrimOp primop_elemAt({ /* Return the first element of a list. */ static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], 0, v); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.head'"); + if (args[0]->listSize() == 0) + state.error( + "'builtins.head' called on an empty list" + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[0], pos); + v = *args[0]->listElems()[0]; } static RegisterPrimOp primop_head({ @@ -3310,9 +3312,9 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.tail'"); if (args[0]->listSize() == 0) - state.error("'tail' called on an empty list").atPos(pos).debugThrow(); + state.error("'builtins.tail' called on an empty list").atPos(pos).debugThrow(); auto list = state.buildList(args[0]->listSize() - 1); for (const auto & [n, v] : enumerate(list)) From 1edf868213a5462b67c3c269f34bf8555c77758a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 16:49:02 +0100 Subject: [PATCH 108/243] rename: nix-tests -> nix-daemon-compat-tests I think I have failed to read the very long version-garbage-like string for the second time now, leaving me oblivious to the crucial info that a test failure happens in the context of an older daemon. --- packaging/hydra.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 81406a249..6109f479e 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -19,11 +19,11 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { pname = - "nix-tests" + "nix-daemon-compat-tests" + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-against-${daemon.version}"; + "-${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From d0b4db924a80d74540b81a6b717aa01498df8ceb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 16:39:48 +0100 Subject: [PATCH 109/243] rename: build-utils-meson -> nix-meson-build-support Fix a footgun. In my case, I had a couple of build ("output") directories sitting around. rm -rf build-* Was confused for a bit why a meson.build file was missing. Probably also helps with autocompletion. I tried meson-build-support first, but I had to add something like a nix- prefix, in order to make meson happy. They've reserved the meson- prefix. --- .../common/meson.build | 0 .../deps-lists/meson.build | 0 .../export-all-symbols/meson.build | 0 .../export/meson.build | 0 .../generate-header/meson.build | 0 .../libatomic/meson.build | 0 .../subprojects/meson.build | 0 .../windows-version/meson.build | 0 src/libcmd/build-utils-meson | 1 - src/libcmd/meson.build | 8 ++++---- src/libcmd/nix-meson-build-support | 1 + src/libcmd/package.nix | 4 ++-- src/libexpr-c/build-utils-meson | 1 - src/libexpr-c/meson.build | 12 ++++++------ src/libexpr-c/nix-meson-build-support | 1 + src/libexpr-c/package.nix | 4 ++-- src/libexpr-test-support/build-utils-meson | 1 - src/libexpr-test-support/meson.build | 12 ++++++------ src/libexpr-test-support/nix-meson-build-support | 1 + src/libexpr-test-support/package.nix | 4 ++-- src/libexpr-tests/build-utils-meson | 1 - src/libexpr-tests/meson.build | 10 +++++----- src/libexpr-tests/nix-meson-build-support | 1 + src/libexpr-tests/package.nix | 4 ++-- src/libexpr/build-utils-meson | 1 - src/libexpr/meson.build | 10 +++++----- src/libexpr/nix-meson-build-support | 1 + src/libexpr/package.nix | 4 ++-- src/libfetchers-tests/build-utils-meson | 1 - src/libfetchers-tests/meson.build | 10 +++++----- src/libfetchers-tests/nix-meson-build-support | 1 + src/libfetchers-tests/package.nix | 4 ++-- src/libfetchers/build-utils-meson | 1 - src/libfetchers/meson.build | 8 ++++---- src/libfetchers/nix-meson-build-support | 1 + src/libfetchers/package.nix | 4 ++-- src/libflake-c/build-utils-meson | 1 - src/libflake-c/meson.build | 12 ++++++------ src/libflake-c/nix-meson-build-support | 1 + src/libflake-c/package.nix | 4 ++-- src/libflake-tests/build-utils-meson | 1 - src/libflake-tests/meson.build | 10 +++++----- src/libflake-tests/nix-meson-build-support | 1 + src/libflake-tests/package.nix | 4 ++-- src/libflake/build-utils-meson | 1 - src/libflake/meson.build | 8 ++++---- src/libflake/nix-meson-build-support | 1 + src/libflake/package.nix | 4 ++-- src/libmain-c/build-utils-meson | 1 - src/libmain-c/meson.build | 12 ++++++------ src/libmain-c/nix-meson-build-support | 1 + src/libmain-c/package.nix | 4 ++-- src/libmain/build-utils-meson | 1 - src/libmain/meson.build | 8 ++++---- src/libmain/nix-meson-build-support | 1 + src/libmain/package.nix | 4 ++-- src/libstore-c/build-utils-meson | 1 - src/libstore-c/meson.build | 12 ++++++------ src/libstore-c/nix-meson-build-support | 1 + src/libstore-c/package.nix | 4 ++-- src/libstore-test-support/build-utils-meson | 1 - src/libstore-test-support/meson.build | 12 ++++++------ .../nix-meson-build-support | 1 + src/libstore-test-support/package.nix | 4 ++-- src/libstore-tests/build-utils-meson | 1 - src/libstore-tests/meson.build | 10 +++++----- src/libstore-tests/nix-meson-build-support | 1 + src/libstore-tests/package.nix | 4 ++-- src/libstore/build-utils-meson | 1 - src/libstore/meson.build | 16 ++++++++-------- src/libstore/nix-meson-build-support | 1 + src/libstore/package.nix | 4 ++-- src/libutil-c/build-utils-meson | 1 - src/libutil-c/meson.build | 12 ++++++------ src/libutil-c/nix-meson-build-support | 1 + src/libutil-c/package.nix | 4 ++-- src/libutil-test-support/build-utils-meson | 1 - src/libutil-test-support/meson.build | 12 ++++++------ src/libutil-test-support/nix-meson-build-support | 1 + src/libutil-test-support/package.nix | 4 ++-- src/libutil-tests/build-utils-meson | 1 - src/libutil-tests/meson.build | 10 +++++----- src/libutil-tests/nix-meson-build-support | 1 + src/libutil-tests/package.nix | 4 ++-- src/libutil/build-utils-meson | 1 - src/libutil/meson.build | 14 +++++++------- src/libutil/nix-meson-build-support | 1 + src/libutil/package.nix | 4 ++-- src/nix/build-utils-meson | 1 - src/nix/meson.build | 12 ++++++------ src/nix/nix-meson-build-support | 1 + src/nix/package.nix | 4 ++-- 92 files changed, 178 insertions(+), 178 deletions(-) rename {build-utils-meson => nix-meson-build-support}/common/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/deps-lists/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/export-all-symbols/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/export/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/generate-header/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/libatomic/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/subprojects/meson.build (100%) rename {build-utils-meson => nix-meson-build-support}/windows-version/meson.build (100%) delete mode 120000 src/libcmd/build-utils-meson create mode 120000 src/libcmd/nix-meson-build-support delete mode 120000 src/libexpr-c/build-utils-meson create mode 120000 src/libexpr-c/nix-meson-build-support delete mode 120000 src/libexpr-test-support/build-utils-meson create mode 120000 src/libexpr-test-support/nix-meson-build-support delete mode 120000 src/libexpr-tests/build-utils-meson create mode 120000 src/libexpr-tests/nix-meson-build-support delete mode 120000 src/libexpr/build-utils-meson create mode 120000 src/libexpr/nix-meson-build-support delete mode 120000 src/libfetchers-tests/build-utils-meson create mode 120000 src/libfetchers-tests/nix-meson-build-support delete mode 120000 src/libfetchers/build-utils-meson create mode 120000 src/libfetchers/nix-meson-build-support delete mode 120000 src/libflake-c/build-utils-meson create mode 120000 src/libflake-c/nix-meson-build-support delete mode 120000 src/libflake-tests/build-utils-meson create mode 120000 src/libflake-tests/nix-meson-build-support delete mode 120000 src/libflake/build-utils-meson create mode 120000 src/libflake/nix-meson-build-support delete mode 120000 src/libmain-c/build-utils-meson create mode 120000 src/libmain-c/nix-meson-build-support delete mode 120000 src/libmain/build-utils-meson create mode 120000 src/libmain/nix-meson-build-support delete mode 120000 src/libstore-c/build-utils-meson create mode 120000 src/libstore-c/nix-meson-build-support delete mode 120000 src/libstore-test-support/build-utils-meson create mode 120000 src/libstore-test-support/nix-meson-build-support delete mode 120000 src/libstore-tests/build-utils-meson create mode 120000 src/libstore-tests/nix-meson-build-support delete mode 120000 src/libstore/build-utils-meson create mode 120000 src/libstore/nix-meson-build-support delete mode 120000 src/libutil-c/build-utils-meson create mode 120000 src/libutil-c/nix-meson-build-support delete mode 120000 src/libutil-test-support/build-utils-meson create mode 120000 src/libutil-test-support/nix-meson-build-support delete mode 120000 src/libutil-tests/build-utils-meson create mode 120000 src/libutil-tests/nix-meson-build-support delete mode 120000 src/libutil/build-utils-meson create mode 120000 src/libutil/nix-meson-build-support delete mode 120000 src/nix/build-utils-meson create mode 120000 src/nix/nix-meson-build-support diff --git a/build-utils-meson/common/meson.build b/nix-meson-build-support/common/meson.build similarity index 100% rename from build-utils-meson/common/meson.build rename to nix-meson-build-support/common/meson.build diff --git a/build-utils-meson/deps-lists/meson.build b/nix-meson-build-support/deps-lists/meson.build similarity index 100% rename from build-utils-meson/deps-lists/meson.build rename to nix-meson-build-support/deps-lists/meson.build diff --git a/build-utils-meson/export-all-symbols/meson.build b/nix-meson-build-support/export-all-symbols/meson.build similarity index 100% rename from build-utils-meson/export-all-symbols/meson.build rename to nix-meson-build-support/export-all-symbols/meson.build diff --git a/build-utils-meson/export/meson.build b/nix-meson-build-support/export/meson.build similarity index 100% rename from build-utils-meson/export/meson.build rename to nix-meson-build-support/export/meson.build diff --git a/build-utils-meson/generate-header/meson.build b/nix-meson-build-support/generate-header/meson.build similarity index 100% rename from build-utils-meson/generate-header/meson.build rename to nix-meson-build-support/generate-header/meson.build diff --git a/build-utils-meson/libatomic/meson.build b/nix-meson-build-support/libatomic/meson.build similarity index 100% rename from build-utils-meson/libatomic/meson.build rename to nix-meson-build-support/libatomic/meson.build diff --git a/build-utils-meson/subprojects/meson.build b/nix-meson-build-support/subprojects/meson.build similarity index 100% rename from build-utils-meson/subprojects/meson.build rename to nix-meson-build-support/subprojects/meson.build diff --git a/build-utils-meson/windows-version/meson.build b/nix-meson-build-support/windows-version/meson.build similarity index 100% rename from build-utils-meson/windows-version/meson.build rename to nix-meson-build-support/windows-version/meson.build diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libcmd/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 1f27c1614..222817c81 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -14,7 +14,7 @@ project('nix-cmd', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -28,7 +28,7 @@ deps_public_maybe_subproject = [ dependency('nix-flake'), dependency('nix-main'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -70,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'built-path.cc', @@ -125,4 +125,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libcmd/nix-meson-build-support b/src/libcmd/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libcmd/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 244179ee4..53e54d2f8 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -39,8 +39,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 5bcca29e0..1556dae51 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -14,7 +14,7 @@ project('nix-expr-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_expr.cc', @@ -72,8 +72,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_expr_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixexprc', @@ -89,4 +89,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-c/nix-meson-build-support b/src/libexpr-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index df49a8bdc..727b3a811 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-test-support/build-utils-meson b/src/libexpr-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 33d9e17a6..64d4fe218 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-expr-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -26,7 +26,7 @@ deps_public_maybe_subproject = [ dependency('nix-expr'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -40,7 +40,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/value/context.cc', @@ -54,8 +54,8 @@ headers = files( 'tests/value/context.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', @@ -73,4 +73,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-test-support/nix-meson-build-support b/src/libexpr-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 7e92d145f..4842f5f17 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-tests/build-utils-meson b/src/libexpr-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index b50c18c9c..f37e85e57 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -14,7 +14,7 @@ project('nix-expr-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr-tests/nix-meson-build-support b/src/libexpr-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 959d6b84e..70e497b7e 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 28318579e..b3c559ba7 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -14,7 +14,7 @@ project('nix-expr', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-fetchers'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') boost = dependency( 'boost', @@ -77,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') parser_tab = custom_target( input : 'parser.y', @@ -121,7 +121,7 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -205,4 +205,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr/nix-meson-build-support b/src/libexpr/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d97e7f3a8..5171d70fd 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -40,8 +40,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers-tests/build-utils-meson b/src/libfetchers-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index fdab6ba6c..3e82c6111 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store-test-support'), @@ -22,10 +22,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers-tests/nix-meson-build-support b/src/libfetchers-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 7b2ba8f2c..e18d644ed 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 07a1178cc..b4408e943 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -41,7 +41,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libfetchers/nix-meson-build-support b/src/libfetchers/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 70973bdb2..7dad00025 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake-c/build-utils-meson b/src/libflake-c/build-utils-meson deleted file mode 120000 index 91937f183..000000000 --- a/src/libflake-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index 00d9650e7..b7669fe97 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -14,7 +14,7 @@ project('nix-flake-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_public_maybe_subproject = [ dependency('nix-store-c'), dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_flake.cc', @@ -73,8 +73,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_flake.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflakec', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake-c/nix-meson-build-support b/src/libflake-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index a70cbf94e..7425d6140 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -21,8 +21,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake-tests/build-utils-meson b/src/libflake-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c494c414e..5c3c58e53 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -14,7 +14,7 @@ project('nix-flake-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -43,7 +43,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flakeref.cc', diff --git a/src/libflake-tests/nix-meson-build-support b/src/libflake-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index b3a8ac466..f1abbb32d 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 2c1a70a18..f9e217729 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -14,7 +14,7 @@ project('nix-flake', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-fetchers'), dependency('nix-expr'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -39,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flake/config.cc', @@ -72,4 +72,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake/nix-meson-build-support b/src/libflake/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake/package.nix b/src/libflake/package.nix index fff481720..92445739f 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 3cb1e4baa..5a5684b8d 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -14,7 +14,7 @@ project('nix-main-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_main.cc', @@ -65,8 +65,8 @@ headers = [config_h] + files( 'nix_api_main.h', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixmainc', @@ -82,4 +82,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain-c/nix-meson-build-support b/src/libmain-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 5522037f3..d65792921 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 6c6298e2b..87fc8b8d2 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -14,7 +14,7 @@ project('nix-main', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') pubsetbuf_test = ''' #include @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-args.cc', @@ -96,4 +96,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain/nix-meson-build-support b/src/libmain/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7e7b80472..7d9d99b61 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 44b5fe11d..1ac331ad0 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -14,7 +14,7 @@ project('nix-store-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_store.cc', @@ -64,8 +64,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` headers += files('nix_api_store_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstorec', @@ -81,4 +81,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-c/nix-meson-build-support b/src/libstore-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 896a1a39f..351d4510a 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-test-support/build-utils-meson b/src/libstore-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 1f230914f..2a07e56ac 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-store-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -37,7 +37,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/derived-path.cc', @@ -56,8 +56,8 @@ headers = files( 'tests/protocol.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', @@ -75,4 +75,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-test-support/nix-meson-build-support b/src/libstore-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 2543049fe..62f783c4f 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-tests/build-utils-meson b/src/libstore-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fc9152f2f..c4e42634d 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -14,7 +14,7 @@ project('nix-store-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite @@ -50,7 +50,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-protocol.cc', diff --git a/src/libstore-tests/nix-meson-build-support b/src/libstore-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 3704d8c5c..4937b5329 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -28,8 +28,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/meson.build b/src/libstore/meson.build index f836b8d4f..12a0e6376 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -15,7 +15,7 @@ project('nix-store', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') run_command('ln', '-s', meson.project_build_root() / '__nothing_link_target', @@ -81,7 +81,7 @@ if host_machine.system() == 'windows' deps_other += [wsock32] endif -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', @@ -131,7 +131,7 @@ if aws_s3.found() endif deps_other += aws_s3 -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -179,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'binary-cache-store.cc', @@ -416,8 +416,8 @@ foreach name, value : cpp_str_defines ] endforeach -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstore', @@ -446,4 +446,4 @@ if host_machine.system() != 'darwin' } endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore/nix-meson-build-support b/src/libstore/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore/package.nix b/src/libstore/package.nix index f04e3b95f..47a203f83 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -32,8 +32,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index d44453676..44cec1afc 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -14,7 +14,7 @@ project('nix-util-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -23,7 +23,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -45,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_util.cc', @@ -60,8 +60,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it. headers += files('nix_api_util_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutilc', @@ -77,4 +77,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/nix-meson-build-support b/src/libutil-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 35533f981..4caf96804 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -19,8 +19,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-test-support/build-utils-meson b/src/libutil-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 4afed01ca..03ae63f1a 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-util-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -22,7 +22,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -34,7 +34,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/hash.cc', @@ -51,8 +51,8 @@ headers = files( 'tests/string_callback.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-util-test-support', @@ -70,4 +70,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-test-support/nix-meson-build-support b/src/libutil-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index c403e762c..19b5d6b77 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-tests/build-utils-meson b/src/libutil-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f59350774..83ac79e92 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -14,7 +14,7 @@ project('nix-util-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-util'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'args.cc', diff --git a/src/libutil-tests/nix-meson-build-support b/src/libutil-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index b099037ee..f06704e26 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 11b4ea592..bbe7872cf 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -14,7 +14,7 @@ project('nix-util', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -22,7 +22,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # Check for each of these functions, and create a define like `#define # HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 @@ -53,7 +53,7 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -120,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'archive.cc', @@ -256,8 +256,8 @@ else subdir('unix') endif -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutil', @@ -278,4 +278,4 @@ if host_machine.system() == 'windows' libraries_private += ['-lws2_32'] endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil/nix-meson-build-support b/src/libutil/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 17a156740..69ebbf726 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -25,8 +25,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson deleted file mode 120000 index 91937f183..000000000 --- a/src/nix/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/meson.build b/src/nix/meson.build index 5c70c8216..1d4840b12 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -15,7 +15,7 @@ project('nix', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') nix_store = dependency('nix-store') @@ -30,10 +30,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') configdata = configuration_data() @@ -63,8 +63,8 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/generate-header') nix_sources = [config_h] + files( 'add-to-store.cc', diff --git a/src/nix/nix-meson-build-support b/src/nix/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/nix/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix index c7b24efce..9bc139c3b 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -22,8 +22,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions ([ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build From 1421420e862434321c46511a3152016e443dd479 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:18:10 +0100 Subject: [PATCH 110/243] test: Move unusual-logging to run only in logging test case --- tests/functional/dependencies.nix | 15 --------------- tests/functional/logging.sh | 3 +++ tests/functional/logging/unusual-logging.nix | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 tests/functional/logging/unusual-logging.nix diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index d177f5b63..4ff29227f 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -33,20 +33,6 @@ let outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - unusual-logging = mkDerivation { - name = "unusual-logging"; - buildCommand = '' - { - echo "@nix 1" - echo "@nix {}" - echo '@nix {"action": null}' - echo '@nix {"action": 123}' - echo '@nix ][' - } >&$NIX_LOG_FD - touch $out - ''; - }; - in mkDerivation { name = "dependencies-top"; @@ -56,7 +42,6 @@ mkDerivation { input1_drv = input1; input2_drv = input2; input0_drv = input0; - unusual_logging_drv = unusual-logging; fod_input_drv = fod_input; meta.description = "Random test package"; } diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index c026ac9c2..bbc1bac96 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -28,3 +28,6 @@ outp="$(nix-build -E \ test -d "$outp" nix log "$outp" + +# Build works despite ill-formed structured build log entries. +expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' diff --git a/tests/functional/logging/unusual-logging.nix b/tests/functional/logging/unusual-logging.nix new file mode 100644 index 000000000..ddb8aa530 --- /dev/null +++ b/tests/functional/logging/unusual-logging.nix @@ -0,0 +1,16 @@ +let + inherit (import ../config.nix) mkDerivation; +in +mkDerivation { + name = "unusual-logging"; + buildCommand = '' + { + echo "@nix 1" + echo "@nix {}" + echo '@nix {"action": null}' + echo '@nix {"action": 123}' + echo '@nix ][' + } >&$NIX_LOG_FD + touch $out + ''; +} From f3c722cab24f7a0de8c3573d25e91749f4f16234 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:20:27 +0100 Subject: [PATCH 111/243] Remove redundant warning: prefix from structured build log warning --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 4c7061016..a5add5565 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -331,7 +331,7 @@ bool handleJSONLogMessage(nlohmann::json & json, return true; } catch (const nlohmann::json::exception &e) { warn( - "warning: Unable to handle a JSON message from %s: %s", + "Unable to handle a JSON message from %s: %s", Uncolored(source), e.what() ); From c783cd22ac1c693871979c4248d7f7b2b7f6bacd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:29:45 +0100 Subject: [PATCH 112/243] tests: Make unusual logging test conditional on fixed daemon version --- tests/functional/logging.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index bbc1bac96..ddb1913ad 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -29,5 +29,7 @@ test -d "$outp" nix log "$outp" -# Build works despite ill-formed structured build log entries. -expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' +if isDaemonNewer "2.26"; then + # Build works despite ill-formed structured build log entries. + expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:' +fi From 6d86839a0265cf06a384499d9d9aac0daf31a8ad Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:34:56 +0100 Subject: [PATCH 113/243] Use version correctly in nix-daemon-compat-tests derivation Previously, a version would be appended to the pname which already contained a version. --- packaging/hydra.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 6109f479e..e4562cd21 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -18,12 +18,12 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { - pname = - "nix-daemon-compat-tests" - + lib.optionalString + pname = "nix-daemon-compat-tests"; + version = + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-with-daemon-${daemon.version}"; + "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From 62d3957268399ef20c517801fd8672b863ceb16d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Dec 2024 17:35:57 +0100 Subject: [PATCH 114/243] Remove dead code from nix-daemon-compat-tests packaging We're not testing against these versions anymore. If we bring that back (I would support that), we could do so in a clean way, by making sure that the packaging we test against has a proper version attribute. --- packaging/hydra.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index e4562cd21..17ba72032 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -19,11 +19,7 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { pname = "nix-daemon-compat-tests"; - version = - lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "${pkgs.nix.version}-with-daemon-${daemon.version}"; + version = "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; From 8768239517dd75955979fc43cee0d411b4e04139 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 9 Dec 2024 11:15:53 -0800 Subject: [PATCH 115/243] Install .nix-channels and nix.conf with 644 instead of 664 As far as I can tell, there's no real reason either of these need to be 664. I'm willing to bet they were just a typo that has lasted for 7 years. While this shouldn't change anything, this is, IMHO, more correct, so let's stop perpetuating the wrong mode! --- scripts/install-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a487d459f..19f787ab5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -690,7 +690,7 @@ place_channel_configuration() { if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + install -m 0644 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" fi } @@ -964,7 +964,7 @@ $NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf + install -m 0644 "$SCRATCH/nix.conf" /etc/nix/nix.conf } From 35dd19d7851fea20eebcd3cd7cbb6e64cdc47a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 15:37:17 +0100 Subject: [PATCH 116/243] ensure clang-format is using the same version on all platforms --- maintainers/flake-module.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1d4e85c8c..1a134b91a 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -12,6 +12,8 @@ hooks = { clang-format = { enable = true; + # https://github.com/cachix/git-hooks.nix/pull/532 + package = pkgs.llvmPackages_latest.clang-tools; excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' From 9b40618d2e455951b699e2e181e4b93ca97e7b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 16:05:37 +0100 Subject: [PATCH 117/243] tests/nixos: disable documentation to improve eval speed we are not testing any nixos modules, so we don't need to generate documentation. This will give us a bit of speed up. --- tests/nixos/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c5f4a23aa..ff1220f35 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -23,6 +23,9 @@ let nix.checkAllErrors = false; # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above? nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli; + + # Evaluate VMs faster + documentation.enable = false; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; From 3a9d64b8e354176b41936b2b9e1d3b20fe0290f2 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 7 Dec 2024 20:46:11 +0100 Subject: [PATCH 118/243] fromJSON/fromTOML: throw if string contains null byte --- src/libexpr/eval.cc | 9 +++++++++ src/libexpr/json-to-value.cc | 2 ++ src/libexpr/primops/fromTOML.cc | 14 +++++++++++--- src/libexpr/value.hh | 2 ++ .../eval-fail-fromJSON-keyWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromJSON-keyWithNullByte.nix | 1 + .../eval-fail-fromJSON-valueWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromJSON-valueWithNullByte.nix | 1 + .../eval-fail-fromTOML-keyWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromTOML-keyWithNullByte.nix | 1 + .../eval-fail-fromTOML-valueWithNullByte.err.exp | 8 ++++++++ .../lang/eval-fail-fromTOML-valueWithNullByte.nix | 1 + 12 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix create mode 100644 tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp create mode 100644 tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 05f58957e..7b6e89534 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3178,5 +3178,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } +void forceNoNullByte(std::string_view s) +{ + if (s.find('\0') != s.npos) { + using namespace std::string_view_literals; + auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); + throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + } +} + } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ac56541a..17cab7ad5 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -50,6 +50,7 @@ class JSONSax : nlohmann::json_sax { public: void key(string_t & name, EvalState & state) { + forceNoNullByte(name); attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; @@ -122,6 +123,7 @@ public: bool string(string_t & val) override { + forceNoNullByte(val); rs->value(state).mkString(val); rs->add(); return true; diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 264046711..404425054 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -28,8 +28,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V auto attrs = state.buildBindings(size); - for(auto & elem : table) + for(auto & elem : table) { + forceNoNullByte(elem.first); visit(attrs.alloc(elem.first), elem.second); + } v.mkAttrs(attrs); } @@ -54,7 +56,11 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V v.mkFloat(toml::get(t)); break;; case toml::value_t::string: - v.mkString(toml::get(t)); + { + auto s = toml::get(t); + forceNoNullByte(s); + v.mkString(s); + } break;; case toml::value_t::local_datetime: case toml::value_t::offset_datetime: @@ -66,7 +72,9 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(toView(s)); + auto str = toView(s); + forceNoNullByte(str); + attrs.alloc("value").mkString(str); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d98161488..88fcae986 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,4 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); +void forceNoNullByte(std::string_view s); + } diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp new file mode 100644 index 000000000..a16192c59 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-keyWithNullByte.nix:1:1: + 1| builtins.fromJSON ''{"a\u0000b": 1}'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix new file mode 100644 index 000000000..ffaa6a97d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"a\u0000b": 1}'' diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp new file mode 100644 index 000000000..c5c08e6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-valueWithNullByte.nix:1:1: + 1| builtins.fromJSON ''"a\u0000b"'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix new file mode 100644 index 000000000..c71ab990d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''"a\u0000b"'' diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp new file mode 100644 index 000000000..dc2180f0b --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-keyWithNullByte.nix:1:1: + 1| builtins.fromTOML ''"a\u0000b" = 1'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix new file mode 100644 index 000000000..b622dc4dc --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''"a\u0000b" = 1'' diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp new file mode 100644 index 000000000..0235692a8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-valueWithNullByte.nix:1:1: + 1| builtins.fromTOML ''k = "a\u0000b"'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix new file mode 100644 index 000000000..183cab6b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''k = "a\u0000b"'' From 41a464c68de54d637f086c9a5f97111feacfa328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 24 Sep 2024 11:18:19 +0200 Subject: [PATCH 119/243] tests/libstore.hh: reformat with clang-format after update --- src/libstore-test-support/tests/libstore.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore-test-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh index 84be52c23..699ba957e 100644 --- a/src/libstore-test-support/tests/libstore.hh +++ b/src/libstore-test-support/tests/libstore.hh @@ -19,12 +19,12 @@ public: protected: LibStoreTest() : store(openStore({ - .variant = - StoreReference::Specified{ - .scheme = "dummy", - }, - .params = {}, - })) + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) { } From b5cdf2e268c7dce5933dbbcc8cae64d170a04b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 10 Dec 2024 05:33:32 +0100 Subject: [PATCH 120/243] mergify: don't use rebase merge method The repository configuration doesn't allow rebase merge method. --- .mergify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index c545bbe6a..5b41c5c8d 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -5,7 +5,6 @@ queue_rules: - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) - check-success=vm_tests - merge_method: rebase batch_size: 5 pull_request_rules: From 6d97d57dc78c156c2dfd1fb830cfbc0152683827 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Dec 2024 16:20:29 +0100 Subject: [PATCH 121/243] Prune unneeded .gitignore entries --- .gitignore | 122 +-------------------------------------- tests/functional/repl.sh | 2 +- 2 files changed, 4 insertions(+), 120 deletions(-) diff --git a/.gitignore b/.gitignore index de1183977..337a7c154 100644 --- a/.gitignore +++ b/.gitignore @@ -1,110 +1,12 @@ -Makefile.config -perl/Makefile.config - -# / -/aclocal.m4 -/autom4te.cache -/precompiled-headers.h.gch -/config.* -/configure -/stamp-h1 -/svn-revision -/libtool -/config/config.* # Default meson build dir /build -# /doc/manual/ -/doc/manual/*.1 -/doc/manual/*.5 -/doc/manual/*.8 -/doc/manual/generated/* -/doc/manual/nix.json -/doc/manual/conf-file.json -/doc/manual/language.json -/doc/manual/xp-features.json -/doc/manual/source/SUMMARY.md -/doc/manual/source/SUMMARY-rl-next.md -/doc/manual/source/store/types/* -!/doc/manual/source/store/types/index.md.in -/doc/manual/source/command-ref/new-cli -/doc/manual/source/command-ref/conf-file.md -/doc/manual/source/command-ref/experimental-features-shortlist.md -/doc/manual/source/contributing/experimental-feature-descriptions.md -/doc/manual/source/language/builtins.md -/doc/manual/source/language/builtin-constants.md -/doc/manual/source/release-notes/rl-next.md - -# /scripts/ -/scripts/nix-profile.sh -/scripts/nix-profile-daemon.sh -/scripts/nix-profile.fish -/scripts/nix-profile-daemon.fish - -# /src/libexpr/ -/src/libexpr/lexer-tab.cc -/src/libexpr/lexer-tab.hh -/src/libexpr/parser-tab.cc -/src/libexpr/parser-tab.hh -/src/libexpr/parser-tab.output -/src/libexpr/nix.tbl -/src/libexpr/tests -/src/libexpr-tests/libnixexpr-tests - -# /src/libfetchers -/src/libfetchers-tests/libnixfetchers-tests - -# /src/libflake -/src/libflake-tests/libnixflake-tests - -# /src/libstore/ -*.gen.* -/src/libstore/tests -/src/libstore-tests/libnixstore-tests - -# /src/libutil/ -/src/libutil/tests -/src/libutil-tests/libnixutil-tests - -/src/nix/nix - -/src/nix/generated-doc - -# /src/nix-env/ -/src/nix-env/nix-env - -# /src/nix-instantiate/ -/src/nix-instantiate/nix-instantiate - -# /src/nix-store/ -/src/nix-store/nix-store - -/src/nix-prefetch-url/nix-prefetch-url - -/src/nix-collect-garbage/nix-collect-garbage - -# /src/nix-channel/ -/src/nix-channel/nix-channel - -# /src/nix-build/ -/src/nix-build/nix-build - -/src/nix-copy-closure/nix-copy-closure - -/src/error-demo/error-demo - -/src/build-remote/build-remote - # /tests/functional/ -/tests/functional/test-tmp /tests/functional/common/subst-vars.sh -/tests/functional/result* /tests/functional/restricted-innocent -/tests/functional/shell -/tests/functional/shell.drv -/tests/functional/repl-result-out /tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer +/tests/functional/nix-shell # /tests/functional/lang/ /tests/functional/lang/*.out @@ -112,27 +14,9 @@ perl/Makefile.config /tests/functional/lang/*.err /tests/functional/lang/*.ast -/perl/lib/Nix/Config.pm -/perl/lib/Nix/Store.cc - -/misc/systemd/nix-daemon.service -/misc/systemd/nix-daemon.socket -/misc/systemd/nix-daemon.conf -/misc/upstart/nix-daemon.conf - outputs/ -*.a -*.o -*.o.tmp -*.so -*.dylib -*.dll -*.exe -*.dep *~ -*.pc -*.plist # GNU Global GPATH @@ -147,8 +31,6 @@ GTAGS compile_commands.json *.compile_commands.json -nix-rust/target - result result-* @@ -163,3 +45,5 @@ result-* # Mac OS .DS_Store + +flake-regressions diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 706e0f5db..59d1f1be0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -309,7 +309,7 @@ runRepl () { for test in $(cd "$testDir/repl"; echo *.in); do test="$(basename "$test" .in)" in="$testDir/repl/$test.in" - actual="$testDir/repl/$test.actual" + actual="$TEST_ROOT/$test.actual" expected="$testDir/repl/$test.expected" (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 From 691b1ea237427845f073fbc790f960023bacdfe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 09:44:36 +0100 Subject: [PATCH 122/243] speed up ci by splitting off eval and build - This speeds up macOS builds from 30 minutes to 11 minutes (3x faster). - Also improve error reporting e.g. printing out what actually failed to build. - As a result we also no longer need swap. --- .github/workflows/ci.yml | 28 +++++++++------------------- scripts/build-checks | 6 ++++++ 2 files changed, 15 insertions(+), 19 deletions(-) create mode 100755 scripts/build-checks diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be96bb484..5fb672769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,14 @@ on: permissions: read-all jobs: + eval: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v30 + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: needs: [check_secrets] @@ -33,25 +41,7 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - if: matrix.os == 'ubuntu-latest' - run: | - free -h - swapon --show - swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') - echo "Found swap: $swap" - sudo swapoff $swap - # resize it (fallocate) - sudo fallocate -l 10G $swap - sudo mkswap $swap - sudo swapon $swap - free -h - ( - while sleep 60; do - free -h - done - ) & - - run: nix --experimental-features 'nix-command flakes' flake check -L - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + - run: scripts/build-checks # Steps to test CI automation in your own fork. # Cachix: diff --git a/scripts/build-checks b/scripts/build-checks new file mode 100755 index 000000000..e0ee70631 --- /dev/null +++ b/scripts/build-checks @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +system=$(nix eval --raw --impure --expr builtins.currentSystem) +nix eval --json ".#checks.$system" --apply builtins.attrNames | \ + jq -r '.[]' | \ + xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }" From 31b3a34e5bb761a9eb84454bbb4d45261f0247ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 10:47:41 +0100 Subject: [PATCH 123/243] binary-tarball: create tarball without listing files This is producing quite a bit of log output in CI. --- scripts/binary-tarball.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 671c8e96e..9de90b7fb 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env '' fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ + tar cfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --mtime='1970-01-01' \ --absolute-names \ From 9f2b9256b6107a2db1f77c6eb7576c7d155e9644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 12:05:49 +0100 Subject: [PATCH 124/243] ci: disable apparmor restrictions For our tests we need to map the root user for some tests. However ubuntu no longer allows this by default: https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fb672769..3af1e8524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,10 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: + # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + if: matrix.os == 'ubuntu-latest' - run: scripts/build-checks # Steps to test CI automation in your own fork. From 4f831e2be5ec5ec82b42c838522ff3d285b88cb0 Mon Sep 17 00:00:00 2001 From: Greg Curtis Date: Thu, 12 Dec 2024 16:29:17 -0500 Subject: [PATCH 125/243] Fix `nix upgrade-nix` profile search Commit cfe66dbec updated `nix upgrade-nix` to use `ExecutablePath::load().find`, which broke the logic for finding the profile associated with the nix executable. The error looks something like: ``` $ sudo -i nix upgrade-nix --debug found Nix in '"/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin/nix"' found profile '/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin' error: directory '"/nix/store/46p1z0w9ad605kky62dr53z4h24k2a5r-nix-2.25.2/bin/nix"' does not appear to be part of a Nix profile ``` This seems to happen for two reasons: 1. The original PATH search resulted in a directory, but `find` returns the path to the executable. Fixed by getting the path's parent. 2. The profile symlink cannot be found because `ExecutablePath::load().find` canonicalizes the executable path. I updated find to normalize the path instead, which seems more in line with how other programs resolve paths. I'm not sure if this affects other callers though. I manually tested this on macOS and Linux, and it seemed to fix upgrading from 2.25.2 to 2.25.3. --- src/libutil/executable-path.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ebd522a41..8d665c7df 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -73,7 +73,7 @@ ExecutablePath::findName(const OsString & exe, std::functionparent_path(); printInfo("found Nix in '%s'", where); From 366611391ec749ff02545ab147a9d4f2792490aa Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Fri, 6 Dec 2024 15:54:47 +0100 Subject: [PATCH 126/243] Implement shellSplitString for proper handling of NIX_SSHOPTS with spaces and quotes --- doc/manual/rl-next/nix-sshopts-parsing.md | 21 +++++ src/libstore/ssh.cc | 13 ++- src/libutil-tests/strings.cc | 107 +++++++++++++++++++++- src/libutil/strings.cc | 104 +++++++++++++++++++++ src/libutil/strings.hh | 7 ++ 5 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 doc/manual/rl-next/nix-sshopts-parsing.md diff --git a/doc/manual/rl-next/nix-sshopts-parsing.md b/doc/manual/rl-next/nix-sshopts-parsing.md new file mode 100644 index 000000000..65fe6f562 --- /dev/null +++ b/doc/manual/rl-next/nix-sshopts-parsing.md @@ -0,0 +1,21 @@ +--- +synopsis: "Improved `NIX_SSHOPTS` parsing for better SSH option handling" +issues: [5181] +prs: [12020] +--- + +The parsing of the `NIX_SSHOPTS` environment variable has been improved to handle spaces and quotes correctly. +Previously, incorrectly split SSH options could cause failures in CLIs like `nix-copy-closure`, +especially when using complex ssh invocations such as `-o ProxyCommand="ssh -W %h:%p ..."`. + +This change introduces a `shellSplitString` function to ensure +that `NIX_SSHOPTS` is parsed in a manner consistent with shell +behavior, addressing common parsing errors. + +For example, the following now works as expected: + +```bash +export NIX_SSHOPTS='-o ProxyCommand="ssh -W %h:%p ..."' +``` + +This update improves the reliability of SSH-related operations using `NIX_SSHOPTS` across Nix CLIs. diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index dec733fd5..116a480ba 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -41,8 +41,17 @@ void SSHMaster::addCommonSSHOpts(Strings & args) { auto state(state_.lock()); - for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) - args.push_back(i); + std::string sshOpts = getEnv("NIX_SSHOPTS").value_or(""); + + try { + std::list opts = shellSplitString(sshOpts); + for (auto & i : opts) + args.push_back(i); + } catch (Error & e) { + e.addTrace({}, "while splitting NIX_SSHOPTS '%s'", sshOpts); + throw; + } + if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { diff --git a/src/libutil-tests/strings.cc b/src/libutil-tests/strings.cc index 8ceb16767..206890bcf 100644 --- a/src/libutil-tests/strings.cc +++ b/src/libutil-tests/strings.cc @@ -2,11 +2,10 @@ #include #include "strings.hh" +#include "error.hh" namespace nix { -using Strings = std::vector; - /* ---------------------------------------------------------------------------- * concatStringsSep * --------------------------------------------------------------------------*/ @@ -345,4 +344,108 @@ RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); } +/* ---------------------------------------------------------------------------- + * shellSplitString + * --------------------------------------------------------------------------*/ + +TEST(shellSplitString, empty) +{ + std::list expected = {}; + + ASSERT_EQ(shellSplitString(""), expected); +} + +TEST(shellSplitString, oneWord) +{ + std::list expected = {"foo"}; + + ASSERT_EQ(shellSplitString("foo"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpaces) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("'foo bar'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpacesAndDoubleQuoteInSingleQuote) +{ + std::list expected = {"foo bar\""}; + + ASSERT_EQ(shellSplitString("'foo bar\"'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithDoubleQuotes) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("\"foo bar\""), expected); +} + +TEST(shellSplitString, twoWords) +{ + std::list expected = {"foo", "bar"}; + + ASSERT_EQ(shellSplitString("foo bar"), expected); +} + +TEST(shellSplitString, twoWordsWithSpacesAndQuotesQuoted) +{ + std::list expected = {"foo bar'", "baz\""}; + + ASSERT_EQ(shellSplitString("\"foo bar'\" 'baz\"'"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedSingleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo '' bar baz ''"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedDoubleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo \"\" bar baz \"\""), expected); +} + +TEST(shellSplitString, singleQuoteDoesNotUseEscapes) +{ + std::list expected = {"foo\\\"bar"}; + + ASSERT_EQ(shellSplitString("'foo\\\"bar'"), expected); +} + +TEST(shellSplitString, doubleQuoteDoesUseEscapes) +{ + std::list expected = {"foo\"bar"}; + + ASSERT_EQ(shellSplitString("\"foo\\\"bar\""), expected); +} + +TEST(shellSplitString, backslashEscapesSpaces) +{ + std::list expected = {"foo bar", "baz", "qux quux"}; + + ASSERT_EQ(shellSplitString("foo\\ bar baz qux\\ quux"), expected); +} + +TEST(shellSplitString, backslashEscapesQuotes) +{ + std::list expected = {"foo\"bar", "baz", "qux'quux"}; + + ASSERT_EQ(shellSplitString("foo\\\"bar baz qux\\'quux"), expected); +} + +TEST(shellSplitString, testUnbalancedQuotes) +{ + ASSERT_THROW(shellSplitString("foo'"), Error); + ASSERT_THROW(shellSplitString("foo\""), Error); + ASSERT_THROW(shellSplitString("foo'bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar\\\""), Error); +} + } // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index c221a43c6..402b7ae98 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -4,6 +4,7 @@ #include "strings-inline.hh" #include "os-string.hh" +#include "error.hh" namespace nix { @@ -48,4 +49,107 @@ template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const s template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + * + * Read https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html for the + * POSIX shell specification, which is technically what we are implementing here. + */ +std::list shellSplitString(std::string_view s) +{ + std::list result; + std::string current; + bool startedCurrent = false; + bool escaping = false; + + auto pushCurrent = [&]() { + if (startedCurrent) { + result.push_back(current); + current.clear(); + startedCurrent = false; + } + }; + + auto pushChar = [&](char c) { + current.push_back(c); + startedCurrent = true; + }; + + auto pop = [&]() { + auto c = s[0]; + s.remove_prefix(1); + return c; + }; + + auto inDoubleQuotes = [&]() { + startedCurrent = true; + // in double quotes, escaping with backslash is only effective for $, `, ", and backslash + while (!s.empty()) { + auto c = pop(); + if (escaping) { + switch (c) { + case '$': + case '`': + case '"': + case '\\': + pushChar(c); + break; + default: + pushChar('\\'); + pushChar(c); + break; + } + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == '"') { + return; + } else { + pushChar(c); + } + } + if (s.empty()) { + throw Error("unterminated double quote"); + } + }; + + auto inSingleQuotes = [&]() { + startedCurrent = true; + while (!s.empty()) { + auto c = pop(); + if (c == '\'') { + return; + } + pushChar(c); + } + if (s.empty()) { + throw Error("unterminated single quote"); + } + }; + + while (!s.empty()) { + auto c = pop(); + if (escaping) { + pushChar(c); + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == ' ' || c == '\t') { + pushCurrent(); + } else if (c == '"') { + inDoubleQuotes(); + } else if (c == '\'') { + inSingleQuotes(); + } else { + pushChar(c); + } + } + + pushCurrent(); + + return result; +} } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 533126be1..c4fd3daa1 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -71,4 +71,11 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + */ +std::list shellSplitString(std::string_view s); } From 7d1c41f74cc02c6a941c690b655d3d5338074bc5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 13:09:03 +0100 Subject: [PATCH 127/243] C API: typo --- src/libutil-c/nix_api_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 43f9fa9dc..5f42641d4 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -47,7 +47,7 @@ extern "C" { */ // Error codes /** - * @brief Type for error codes in the NIX system + * @brief Type for error codes in the Nix system * * This type can have one of several predefined constants: * - NIX_OK: No error occurred (0) From 80ee736b02bb3f400e656f65e3beb1b80a932cf7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 13:05:46 +0100 Subject: [PATCH 128/243] C API: document nix_store_open NULL URI tl;dr NULL is better than "auto" --- src/libstore-c/nix_api_store.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 282ccc285..60a287559 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -48,12 +48,27 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context); * Store instances may share state and resources behind the scenes. * * @param[out] context Optional, stores error information - * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * + * @param[in] uri @parblock + * URI of the Nix store, copied. + * + * If `NULL`, the store from the settings will be used. + * Note that `"auto"` holds a strange middle ground, reading part of the general environment, but not all of it. It + * ignores `NIX_REMOTE` and the `store` option. For this reason, `NULL` is most likely the better choice. + * + * For supported store URLs, see [*Store URL format* in the Nix Reference * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). - * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", - * "https://s3.local"}}. See [*Store Types* in the Nix Reference - * Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * + * @param[in] params @parblock + * optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. + * + * See [*Store Types* in the Nix Reference Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * * @return a Store pointer, NULL in case of errors + * * @see nix_store_free */ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); From 472912f7ca9dbf90fdee18bc0a9b0994974c6660 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Dec 2024 17:43:01 +0100 Subject: [PATCH 129/243] C API: Add nix_store_get_storedir --- src/libstore-c/nix_api_store.cc | 11 ++++++++++ src/libstore-c/nix_api_store.h | 13 +++++++++++- src/libstore-tests/meson.build | 5 ++++- src/libstore-tests/nix_api_store.cc | 33 +++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fb7391276..6e5dd559f 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -67,6 +67,17 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string NIXC_CATCH_ERRS } +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return call_nix_get_string_callback(store->ptr->storeDir, callback, user_data); + } + NIXC_CATCH_ERRS +} + nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 60a287559..d421ee4d3 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -93,7 +93,18 @@ void nix_store_free(Store * store); */ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); -// returns: owned StorePath* +/** + * @brief get the storeDir of a Nix store, typically `"/nix/store"` + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + /** * @brief Parse a Nix store path into a StorePath * diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index c4e42634d..b706fa12c 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -16,8 +16,10 @@ cxx = meson.get_compiler('cpp') subdir('nix-meson-build-support/deps-lists') +nix_store = dependency('nix-store') + deps_private_maybe_subproject = [ - dependency('nix-store'), + nix_store, dependency('nix-store-c'), dependency('nix-store-test-support'), ] @@ -90,6 +92,7 @@ this_exe = executable( include_directories : include_dirs, # TODO: -lrapidcheck, see ../libutil-support/build.meson link_args: linker_export_flags + ['-lrapidcheck'], + cpp_args : [ '-DNIX_STORE_DIR="' + nix_store.get_variable('storedir') + '"' ], # get main from gtest install : true, ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 7c6ec0780..d47cd1dfd 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -24,6 +24,39 @@ TEST_F(nix_api_store_test, nix_store_get_uri) ASSERT_STREQ("local", str.c_str()); } +TEST_F(nix_api_util_context, nix_store_get_storedir_default) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles + GTEST_SKIP(); + } + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, nullptr, nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(NIX_STORE_DIR, str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_get_storedir) +{ + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(nixStoreDir.c_str(), str.c_str()); +} + TEST_F(nix_api_store_test, InvalidPathFails) { nix_store_parse_path(ctx, store, "invalid-path"); From 2a981689423a531f5462f51aea17774ed4f17d0d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Dec 2024 18:51:11 +0100 Subject: [PATCH 130/243] C API: Add nix_store_real_path --- src/libstore-c/nix_api_store.cc | 12 +++++ src/libstore-c/nix_api_store.h | 20 ++++++++ src/libstore-tests/nix_api_store.cc | 80 +++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 6e5dd559f..bc306e0d0 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -100,6 +100,18 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * NIXC_CATCH_ERRS_RES(false); } +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->toRealPath(path->path); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index d421ee4d3..e55bc3f59 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -149,6 +149,26 @@ void nix_store_path_free(StorePath * p); * @return true or false, error info in context */ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); + +/** + * @brief Get the physical location of a store path + * + * A store may reside at a different location than its `storeDir` suggests. + * This situation is called a relocated store. + * Relocated stores are used during NixOS installation, as well as in restricted computing environments that don't offer + * a writable `/nix/store`. + * + * Not all types of stores support this operation. + * + * @param[in] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path the path to get the real path from + * @param[in] callback called with the real path + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data); + // nix_err nix_store_ensure(Store*, const char*); // nix_err nix_store_build_paths(Store*); /** diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index d47cd1dfd..a8b7b8e5f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -119,4 +119,84 @@ TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); } +TEST_F(nix_api_store_test, nix_store_real_path) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((nixStoreDir + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); } + +TEST_F(nix_api_util_context, nix_store_real_path_relocated) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // Can't open default store from within sandbox + GTEST_SKIP(); + } + auto tmp = nix::createTempDir(); + std::string storeRoot = tmp + "/store"; + std::string stateDir = tmp + "/state"; + std::string logDir = tmp + "/log"; + const char * rootkv[] = {"root", storeRoot.c_str()}; + const char * statekv[] = {"state", stateDir.c_str()}; + const char * logkv[] = {"log", logDir.c_str()}; + // const char * rokv[] = {"read-only", "true"}; + const char ** kvs[] = {rootkv, statekv, logkv, NULL}; + + nix_libstore_init(ctx); + assert_ctx_ok(); + + Store * store = nix_store_open(ctx, "local", kvs); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string nixStoreDir; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(nixStoreDir)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(NIX_STORE_DIR, nixStoreDir.c_str()); + + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((storeRoot + NIX_STORE_DIR + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); +} + +TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // TODO: override NIX_CACHE_HOME? + // skipping test in sandbox because narinfo cache can't be written + GTEST_SKIP(); + } + + Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string path_raw = std::string(NIX_STORE_DIR) + PATH_SUFFIX; + StorePath * path = nix_store_parse_path(ctx, store, path_raw.c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(path_raw.c_str(), rp.c_str()); +} + +} // namespace nixC From da7f7ba810e8fbcdc28bca490b5a5c618b1cef2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 14 Dec 2024 12:55:45 +0100 Subject: [PATCH 131/243] functional-tests: skip tests if the kernel restricts unprivileged user namespaces Update tests/functional/common/functions.sh Co-authored-by: Robert Hensing --- .../build-remote-trustless-should-fail-0.sh | 1 + tests/functional/build-remote-trustless.sh | 1 + tests/functional/build-remote.sh | 2 ++ tests/functional/chroot-store.sh | 1 + tests/functional/common/functions.sh | 11 +++++++++++ tests/functional/linux-sandbox.sh | 1 + tests/functional/local-overlay-store/bad-uris.sh | 2 +- tests/functional/local-overlay-store/common.sh | 4 ---- tests/functional/nested-sandboxing.sh | 1 + tests/functional/nested-sandboxing/command.sh | 1 + tests/functional/shell.sh | 1 + tests/functional/supplementary-groups.sh | 2 +- 12 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 4eccb73e0..3401de1b0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -8,6 +8,7 @@ TODO_NixOS restartDaemon requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh index c498d46c3..9f91a91a9 100644 --- a/tests/functional/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -5,6 +5,7 @@ # shellcheck disable=SC2154 requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "$busybox" =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 1a5334577..3231341cb 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -3,6 +3,7 @@ : "${file?must be defined by caller (remote building test case using this)}" requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" # Avoid store dir being inside sandbox build-dir @@ -27,6 +28,7 @@ builders=( chmod -R +w "$TEST_ROOT/machine"* || true rm -rf "$TEST_ROOT/machine"* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 46e91f0aa..ccde3e90b 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -40,6 +40,7 @@ EOF cp simple.nix shell.nix simple.builder.sh "${config_nix}" "$flakeDir/" TODO_NixOS + requiresUnprivilegedUserNamespaces outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 7195149cb..bf3dd2ca8 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -345,4 +345,15 @@ count() { trap onError ERR +requiresUnprivilegedUserNamespaces() { + if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then + skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests." + fi +} + +execUnshare () { + requiresUnprivilegedUserNamespaces + exec unshare --mount --map-root-user "$SHELL" "$@" +} + fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 81ef36237..abb635f11 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -9,6 +9,7 @@ TODO_NixOS clearStore requireSandboxSupport +requiresUnprivilegedUserNamespaces # Note: we need to bind-mount $SHELL into the chroot. Currently we # only support the case where $SHELL is in the Nix store, because diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index b7930e32e..f0c6a151c 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -19,7 +19,7 @@ TODO_NixOS for i in "${storesBad[@]}"; do echo $i - unshare --mount --map-root-user bash < Date: Sat, 14 Dec 2024 17:17:29 +0100 Subject: [PATCH 132/243] installer: make sure we can always test the installer in ci and locally Just now there is a dependency on cachix, which means we cannot test the installer in CI if forks do not have the necessary secrets set up. We replace this with a simple http server that serves the installer and can be both used in CI and locally. --- .github/workflows/ci.yml | 119 +++++++------------ flake.nix | 4 +- packaging/hydra.nix | 13 +- scripts/prepare-installer-for-github-actions | 13 +- scripts/serve-installer-for-github-actions | 22 ++++ 5 files changed, 82 insertions(+), 89 deletions(-) create mode 100755 scripts/serve-installer-for-github-actions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3af1e8524..addafb9f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,6 @@ jobs: - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: - needs: [check_secrets] strategy: fail-fast: false matrix: @@ -34,72 +33,21 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 if: matrix.os == 'ubuntu-latest' - run: scripts/build-checks - - # Steps to test CI automation in your own fork. - # Cachix: - # 1. Sign-up for https://www.cachix.org/ - # 2. Create a cache for $githubuser-nix-install-tests - # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN - # Dockerhub: - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 + - run: scripts/prepare-installer-for-github-actions + - name: Upload installer tarball + uses: actions/upload-artifact@v4 with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v15 - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - cachixArgs: '-v' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions + name: installer-${{matrix.os}} + path: out/* installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' + needs: [tests] strategy: fail-fast: false matrix: @@ -107,11 +55,18 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - name: Download installer tarball + uses: actions/download-artifact@v4 + with: + name: installer-${{matrix.os}} + path: out + - name: Serving installer + id: serving_installer + run: ./scripts/serve-installer-for-github-actions - uses: cachix/install-nix-action@v30 with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + install_url: 'http://localhost:8126/install' + install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-latest' - run: brew install fish @@ -123,32 +78,50 @@ jobs: - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + # Steps to test CI automation in your own fork. + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork + check_secrets: + permissions: + contents: none + name: Check Docker secrets present for installer tests + runs-on: ubuntu-latest + outputs: + docker: ${{ steps.secret.outputs.docker }} + steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + docker_push_image: - needs: [check_secrets, tests, vm_tests] + needs: [tests, vm_tests, check_secrets] permissions: contents: read packages: write if: >- + needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' + github.ref_name == 'master' runs-on: ubuntu-latest steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION diff --git a/flake.nix b/flake.nix index 794736af4..a5e68609e 100644 --- a/flake.nix +++ b/flake.nix @@ -186,7 +186,7 @@ }; checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -237,6 +237,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; + binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 17ba72032..5b1e47559 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -123,15 +123,10 @@ in self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; + + installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix { + tarballs = [ self.hydraJobs.binaryTarball.${system} ]; + }); # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 4b994a753..0fbecf25c 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -1,10 +1,11 @@ #!/usr/bin/env bash -set -e +set -euo pipefail -script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo "$script" | cut -b12-43 -) +nix build -L ".#installerScriptForGHA" ".#binaryTarball" -installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install - -echo "::set-output name=installerURL::$installerURL" +mkdir -p out +cp ./result/install "out/install" +name="$(basename "$(realpath ./result-1)")" +# everything before the first dash +cp -r ./result-1 "out/${name%%-*}" diff --git a/scripts/serve-installer-for-github-actions b/scripts/serve-installer-for-github-actions new file mode 100755 index 000000000..2efd2aa32 --- /dev/null +++ b/scripts/serve-installer-for-github-actions @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [[ ! -d out ]]; then + echo "run prepare-installer-for-github-actions first" + exit 1 +fi +cd out +PORT=${PORT:-8126} +nohup python -m http.server "$PORT" >/dev/null 2>&1 & +pid=$! + +while ! curl -s "http://localhost:$PORT"; do + sleep 1 + if ! kill -0 $pid; then + echo "Failed to start http server" + exit 1 + fi +done + +echo 'To install nix, run the following command:' +echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT" From df3eb4f3f6e80fa3bdcd17f36ec6b0a9156825fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 18:45:03 +0100 Subject: [PATCH 133/243] mergify: updated required tests --- .mergify.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 5b41c5c8d..70fccae49 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,6 +4,8 @@ queue_rules: merge_conditions: - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) - check-success=vm_tests batch_size: 5 From ccaa4c259ad33d5378e0641f9bb891430bf9bbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 21:04:26 +0100 Subject: [PATCH 134/243] upgrade-nix: don't double quote path on error the format error already adds quotes. --- src/nix/upgrade-nix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a4722abf5..77fb798a7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -126,7 +126,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + throw Error("directory %s does not appear to be part of a Nix profile", where); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); From 7ddf7300b52c8500718ca1fbaca76bab017f6edf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Dec 2024 11:45:26 +0100 Subject: [PATCH 135/243] Ignore local registries for lock file generation When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. --- doc/manual/rl-next/ignore-local-registries.md | 22 +++++++++++++++++++ src/libfetchers/registry.cc | 4 +++- src/libfetchers/registry.hh | 9 +++++++- src/libflake/flake/flake.cc | 8 ++++++- src/libflake/flake/flakeref.cc | 5 +++-- src/libflake/flake/flakeref.hh | 5 ++++- src/nix/flake.md | 3 ++- src/nix/registry.md | 2 ++ 8 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 doc/manual/rl-next/ignore-local-registries.md diff --git a/doc/manual/rl-next/ignore-local-registries.md b/doc/manual/rl-next/ignore-local-registries.md new file mode 100644 index 000000000..8d5e333dd --- /dev/null +++ b/doc/manual/rl-next/ignore-local-registries.md @@ -0,0 +1,22 @@ +--- +synopsis: "Flake lock file generation now ignores local registries" +prs: [12019] +--- + +When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. + +This avoids accidents where users have local registry overrides that map `nixpkgs` to a `path:` flake in the local file system, which then end up in committed lock files pushed to other users. + +In the future, we may remove the use of the registry during lock file generation altogether. It's better to explicitly specify the URL of a flake input. For example, instead of +```nix +{ + outputs = { self, nixpkgs }: { ... }; +} +``` +write +```nix +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + outputs = { self, nixpkgs }: { ... }; +} +``` diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index c761028ab..171afcea7 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -178,7 +178,8 @@ Registries getRegistries(const Settings & settings, ref store) std::pair lookupInRegistries( ref store, - const Input & _input) + const Input & _input, + const RegistryFilter & filter) { Attrs extraAttrs; int n = 0; @@ -190,6 +191,7 @@ std::pair lookupInRegistries( if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(*input.settings, store)) { + if (filter && !filter(registry->type)) continue; // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 0d68ac395..8f47e1590 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -65,8 +65,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs); +using RegistryFilter = std::function; + +/** + * Rewrite a flakeref using the registries. If `filter` is set, only + * use the registries for which the filter function returns true. + */ std::pair lookupInRegistries( ref store, - const Input & input); + const Input & input, + const RegistryFilter & filter = {}); } diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 19b622a34..01cd8db65 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -54,7 +54,13 @@ static std::tuple fetchOrSubstituteTree( fetched.emplace(originalRef.fetchTree(state.store)); } else { if (allowLookup) { - resolvedRef = originalRef.resolve(state.store); + resolvedRef = originalRef.resolve( + state.store, + [](fetchers::Registry::RegistryType type) { + /* Only use the global registry and CLI flags + to resolve indirect flakerefs. */ + return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; + }); auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); flakeCache.push_back({resolvedRef, *fetchedResolved}); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 9616fe0ea..ab882fdab 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -3,7 +3,6 @@ #include "url.hh" #include "url-parts.hh" #include "fetchers.hh" -#include "registry.hh" namespace nix { @@ -36,7 +35,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -FlakeRef FlakeRef::resolve(ref store) const +FlakeRef FlakeRef::resolve( + ref store, + const fetchers::RegistryFilter & filter) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 1064538a7..80013e87e 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "fetchers.hh" #include "outputs-spec.hh" +#include "registry.hh" namespace nix { @@ -57,7 +58,9 @@ struct FlakeRef fetchers::Attrs toAttrs() const; - FlakeRef resolve(ref store) const; + FlakeRef resolve( + ref store, + const fetchers::RegistryFilter & filter = {}) const; static FlakeRef fromAttrs( const fetchers::Settings & fetchSettings, diff --git a/src/nix/flake.md b/src/nix/flake.md index 1e9895f6e..1028dc807 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -165,7 +165,8 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `indirect`: *The default*. Indirection through the flake registry. +* `indirect`: *The default*. These are symbolic references to flakes + that are looked up in [the flake registries](./nix3-registry.md). These have the form ``` diff --git a/src/nix/registry.md b/src/nix/registry.md index bd3575d1b..d6f8af5e9 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -34,6 +34,8 @@ highest precedence: * Overrides specified on the command line using the option `--override-flake`. +Note that the system and user registries are not used to resolve flake references in `flake.nix`. They are only used to resolve flake references on the command line. + # Registry format A registry is a JSON file with the following format: From d8117c8c0de0a533a42b563ed01e67976368da09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:46:02 +0100 Subject: [PATCH 136/243] libexpr: always initialize displacement Otherwise the value is undefined in some cases. --- src/libexpr/nixexpr.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 2950ff1fd..a7ad580d2 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -168,7 +168,7 @@ struct ExprVar : Expr the set stored in the environment that is `level` levels up from the current one.*/ Level level; - Displacement displ; + Displacement displ = 0; ExprVar(Symbol name) : name(name) { }; ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; @@ -242,7 +242,7 @@ struct ExprAttrs : Expr Kind kind; Expr * e; PosIdx pos; - Displacement displ; // displacement + Displacement displ = 0; // displacement AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) : kind(kind), e(e), pos(pos) { }; AttrDef() { }; From d1f20e3510ef39bc36287773cff35383e824ebc0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Dec 2024 14:58:25 +0100 Subject: [PATCH 137/243] Make FetchedFlake a struct --- src/libflake/flake/flake.cc | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index bc62e0bc5..0bdaeb789 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -21,7 +21,12 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +struct FetchedFlake +{ + FlakeRef lockedRef; + StorePath storePath; +}; + typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -32,7 +37,7 @@ static std::optional lookupInFlakeCache( for (auto & i : flakeCache) { if (flakeRef == i.first) { debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.second); + flakeRef, i.first, i.second.lockedRef); return i.second; } } @@ -51,7 +56,8 @@ static std::tuple fetchOrSubstituteTree( if (!fetched) { if (originalRef.input.isDirect()) { - fetched.emplace(originalRef.fetchTree(state.store)); + auto [storePath, lockedRef] = originalRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { if (allowLookup) { resolvedRef = originalRef.resolve( @@ -61,10 +67,12 @@ static std::tuple fetchOrSubstituteTree( to resolve indirect flakerefs. */ return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; }); - auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); - if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); - flakeCache.push_back({resolvedRef, *fetchedResolved}); - fetched.emplace(*fetchedResolved); + fetched = lookupInFlakeCache(flakeCache, originalRef); + if (!fetched) { + auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + } + flakeCache.push_back({resolvedRef, *fetched}); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); @@ -73,16 +81,14 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [storePath, lockedRef] = *fetched; - debug("got tree '%s' from '%s'", - state.store->printStorePath(storePath), lockedRef); + state.store->printStorePath(fetched->storePath), fetched->lockedRef); - state.allowPath(storePath); + state.allowPath(fetched->storePath); - assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(storePath), resolvedRef, lockedRef}; + return {fetched->storePath, resolvedRef, fetched->lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) From d2e1d4916a7a74ba75dbbdff0f696acb0859dbfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Dec 2024 15:30:55 +0100 Subject: [PATCH 138/243] lookupInFlakeCache(): Fix O(n) time lookup --- src/libfetchers/fetchers.hh | 5 +++++ src/libflake/flake/flake.cc | 21 ++++++++------------- src/libflake/flake/flakeref.hh | 2 ++ src/libutil/types.hh | 7 +++---- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b28ec4568..94b2320f9 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -104,6 +104,11 @@ public: bool operator ==(const Input & other) const noexcept; + auto operator <=>(const Input & other) const + { + return attrs <=> other.attrs; + } + bool contains(const Input & other) const; /** diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 0bdaeb789..29090b900 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -27,22 +27,17 @@ struct FetchedFlake StorePath storePath; }; -typedef std::vector> FlakeCache; +typedef std::map FlakeCache; static std::optional lookupInFlakeCache( const FlakeCache & flakeCache, const FlakeRef & flakeRef) { - // FIXME: inefficient. - for (auto & i : flakeCache) { - if (flakeRef == i.first) { - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.lockedRef); - return i.second; - } - } - - return std::nullopt; + auto i = flakeCache.find(flakeRef); + if (i == flakeCache.end()) return std::nullopt; + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i->first, i->second.lockedRef); + return i->second; } static std::tuple fetchOrSubstituteTree( @@ -72,13 +67,13 @@ static std::tuple fetchOrSubstituteTree( auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } - flakeCache.push_back({resolvedRef, *fetched}); + flakeCache.insert_or_assign(resolvedRef, *fetched); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); } } - flakeCache.push_back({originalRef, *fetched}); + flakeCache.insert_or_assign(originalRef, *fetched); } debug("got tree '%s' from '%s'", diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 80013e87e..2ba01c72b 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -49,6 +49,8 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; + auto operator <=>(const FlakeRef &) const = default; + FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) { } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 325e3ea73..f30dd910e 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -43,10 +43,9 @@ template struct Explicit { T t; - bool operator ==(const Explicit & other) const - { - return t == other.t; - } + bool operator ==(const Explicit & other) const = default; + + auto operator <=>(const Explicit & other) const = default; }; From b167e2c415684518626265abfbdb3b756f04ee1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Dec 2024 20:59:59 +0100 Subject: [PATCH 139/243] Work around clang/libc++ issue --- src/libfetchers/fetchers.hh | 4 ++-- src/libflake/flake/flakeref.hh | 5 ++++- src/libutil/types.hh | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 94b2320f9..ff04d5551 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -104,9 +104,9 @@ public: bool operator ==(const Input & other) const noexcept; - auto operator <=>(const Input & other) const + bool operator <(const Input & other) const { - return attrs <=> other.attrs; + return attrs < other.attrs; } bool contains(const Input & other) const; diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 2ba01c72b..ec755399d 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -49,7 +49,10 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; - auto operator <=>(const FlakeRef &) const = default; + bool operator <(const FlakeRef & other) const + { + return std::tie(input, subdir) < std::tie(other.input, other.subdir); + } FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index f30dd910e..9f5c75827 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -45,7 +45,10 @@ struct Explicit { bool operator ==(const Explicit & other) const = default; - auto operator <=>(const Explicit & other) const = default; + bool operator <(const Explicit & other) const + { + return t < other.t; + } }; From 08361f031d65703aa35cb3b7e7715ff63256e311 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Dec 2024 16:49:48 +0100 Subject: [PATCH 140/243] EvalState::realiseContext(): Allow access to the entire closure Fixes #11030. --- src/libexpr/eval.cc | 15 +++++++++++---- src/libexpr/eval.hh | 5 +++++ src/libexpr/primops.cc | 8 +++----- tests/functional/import-from-derivation.nix | 21 ++++++++++++++++++++- tests/functional/import-from-derivation.sh | 5 +++++ 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7b6e89534..fe5f05ab8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -347,6 +347,16 @@ void EvalState::allowPath(const StorePath & storePath) rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } +void EvalState::allowClosure(const StorePath & storePath) +{ + if (!rootFS.dynamic_pointer_cast()) return; + + StorePathSet closure; + store->computeFSClosure(storePath, closure); + for (auto & p : closure) + allowPath(p); +} + void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); @@ -3099,10 +3109,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat allowPath(path.path.abs()); if (store->isInStore(path.path.abs())) { try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); - for (auto & p : closure) - allowPath(p); + allowClosure(store->toStorePath(path.path.abs()).first); } catch (InvalidPath &) { } } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3ac3c8a8a..84b7d823c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,6 +400,11 @@ public: */ void allowPath(const StorePath & storePath); + /** + * Allow access to the closure of a store path. + */ + void allowClosure(const StorePath & storePath); + /** * Allow access to a store path and return it as a string. */ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c5c6ea9b..c1f1cf4c1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -119,11 +119,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); if (isIFD) { - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); - } + /* Allow access to the output closures of this derivation. */ + for (auto & outputPath : outputsToCopyAndAllow) + allowClosure(outputPath); } return res; diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index cc53451cf..770dd86cf 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ; rec { bar = mkDerivation { @@ -30,4 +30,23 @@ rec { echo -n BLA$(cat $src) > $out ''; }; + + step1 = mkDerivation { + name = "step1"; + buildCommand = '' + mkdir -p $out + echo 'foo' > $out/bla + ''; + }; + + addPathExpr = mkDerivation { + name = "add-path"; + inherit step1; + buildCommand = '' + mkdir -p $out + echo "builtins.path { path = \"$step1\"; sha256 = \"7ptL+pnrZXnSa5hwwB+2SXTLkcSb5264WGGokN8OXto=\"; }" > $out/default.nix + ''; + }; + + importAddPathExpr = import addPathExpr; } diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index 83ef92a6f..a00761235 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -6,6 +6,8 @@ TODO_NixOS clearStoreIfPossible +export NIX_PATH=config="${config_nix}" + if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 @@ -15,6 +17,9 @@ outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] +# Check that we can have access to the entire closure of a derivation output. +nix build --no-link --restrict-eval -I src=. -f ./import-from-derivation.nix importAddPathExpr -v + # FIXME: the next tests are broken on CA. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then exit 0 From cd002ae6dda7be6e339d7e573ee122b5cdc50861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:48:23 +0200 Subject: [PATCH 141/243] flake.nix: switch to nixpkgs 24.11 --- flake.lock | 20 ++++++++++---------- flake.nix | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/flake.lock b/flake.lock index b5d0b881c..10957359a 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", "type": "github" }, "original": { @@ -48,11 +48,11 @@ ] }, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", "type": "github" }, "original": { @@ -80,16 +80,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723688146, - "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", + "lastModified": 1734359947, + "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", + "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05", + "ref": "release-24.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a5e68609e..ba3cd3b82 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.11"; + inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 9131905185129034aa12839809ffd8f843107940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:51:29 +0200 Subject: [PATCH 142/243] use libgit2 from nixpkgs --- flake.lock | 24 +++--------------------- flake.nix | 4 +--- packaging/dependencies.nix | 2 -- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/flake.lock b/flake.lock index 10957359a..ce484a67a 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -61,23 +61,6 @@ "type": "github" } }, - "libgit2": { - "flake": false, - "locked": { - "lastModified": 1715853528, - "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", - "owner": "libgit2", - "repo": "libgit2", - "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", - "type": "github" - }, - "original": { - "owner": "libgit2", - "ref": "v1.8.1", - "repo": "libgit2", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1734359947, @@ -131,7 +114,6 @@ "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", - "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index ba3cd3b82..156eb71cd 100644 --- a/flake.nix +++ b/flake.nix @@ -6,7 +6,6 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; @@ -19,7 +18,7 @@ inputs.git-hooks-nix.inputs.flake-compat.follows = ""; inputs.git-hooks-nix.inputs.gitignore.follows = ""; - outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, ... }: let @@ -164,7 +163,6 @@ if prev.stdenv.hostPlatform.system == "i686-linux" then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) else prev.pre-commit; - }; in { diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index a8005ce16..78451e549 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -140,8 +140,6 @@ scope: { }); libgit2 = pkgs.libgit2.overrideAttrs (attrs: { - src = inputs.libgit2; - version = inputs.libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; nativeBuildInputs = attrs.nativeBuildInputs or [] From 1a8bd84f558c44feff234652669e1474a78289fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:53:06 +0200 Subject: [PATCH 143/243] remove upstreamed busybox-sandbox-shell --- packaging/dependencies.nix | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 78451e549..785480921 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -167,30 +167,6 @@ scope: { ]; }); - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - # TODO change in Nixpkgs, Windows works fine. First commit of # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. toml11 = pkgs.toml11.overrideAttrs (old: { From 81b6b79a5641aaa29244be29bc506e86de51c87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:53:50 +0200 Subject: [PATCH 144/243] remove upstreamed libseccomp --- packaging/dependencies.nix | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 785480921..531052107 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -114,14 +114,6 @@ scope: { requiredSystemFeatures = [ ]; }; - libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = pkgs.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - boehmgc = pkgs.boehmgc.override { enableLargeConfig = true; }; From b5ad051b6c705fe9a6fe442371d9f19257d68724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:56:01 +0200 Subject: [PATCH 145/243] remove upstreamed toml11 package override --- packaging/dependencies.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 531052107..2c7cf701f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -159,12 +159,6 @@ scope: { ]; }); - # TODO change in Nixpkgs, Windows works fine. First commit of - # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. - toml11 = pkgs.toml11.overrideAttrs (old: { - meta.platforms = lib.platforms.all; - }); - inherit resolvePath filesetToSource; mkMesonDerivation = From cf0ba0d20ebdf190141ddc1a114a4a1cf1988bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 25 Sep 2024 14:13:33 +0200 Subject: [PATCH 146/243] netbsd: disable cross-compilation Cross-compilation of curl is broken in nixpkgs. Therefore we disable it until nixpkgs fixes the underlying issues. --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 156eb71cd..104cbc34b 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,8 @@ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" "riscv64-unknown-linux-gnu" - "x86_64-unknown-netbsd" + # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 + # "x86_64-unknown-netbsd" "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; From 2512619cb6b7b5fad7c5024a985996e6c875dfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 3 Dec 2024 15:51:29 +0100 Subject: [PATCH 147/243] switch to lowdown-unsandboxed package This is needed for macos support as the sandboxed version of lowdown doesn't work in the nix sandbox. --- doc/manual/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 2e6fcede3..f8133f2e1 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -3,7 +3,7 @@ , meson , ninja -, lowdown +, lowdown-unsandboxed , mdbook , mdbook-linkcheck , jq @@ -42,7 +42,7 @@ mkMesonDerivation (finalAttrs: { passthru.externalNativeBuildInputs = [ meson ninja - (lib.getBin lowdown) + (lib.getBin lowdown-unsandboxed) mdbook mdbook-linkcheck jq From 20ee83fffde3d5892cf012dedeacaf11ce45cf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 16:03:36 +0100 Subject: [PATCH 148/243] tests/nixos: disable nixos-option --- tests/nixos/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index ff1220f35..a9a438a32 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -26,6 +26,8 @@ let # Evaluate VMs faster documentation.enable = false; + # this links against nix and might break with our git version. + system.tools.nixos-option.enable = false; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; From 51463d228010360e91366e2fc06f8d54ccfc9288 Mon Sep 17 00:00:00 2001 From: Jason Yundt Date: Mon, 16 Dec 2024 13:55:41 -0500 Subject: [PATCH 149/243] Fix flake.lock/flake.nix mix-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/nix/flake.md describes the format of flake.lock files. Before this change, it said that the original field was “The original input specification from `flake.lock`[…]” The original input specification is in flake.nix, not flake.lock. --- src/nix/flake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 1028dc807..fa17375df 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -654,7 +654,7 @@ following fields: * `inputs`: The dependencies of this node, as a mapping from input names (e.g. `nixpkgs`) to node labels (e.g. `n2`). -* `original`: The original input specification from `flake.lock`, as a +* `original`: The original input specification from `flake.nix`, as a set of `builtins.fetchTree` arguments. * `locked`: The locked input specification, as a set of From 331bf3e2613d2b1f730f3867dd32bd0dc1ca43a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Nov 2024 16:55:27 +0100 Subject: [PATCH 150/243] Git fetcher: Calculate a fingerprint for dirty workdirs This restores evaluation caching for dirty Git workdirs. --- src/libfetchers/git-utils.cc | 18 ++++++++++++++++-- src/libfetchers/git-utils.hh | 7 ++++++- src/libfetchers/git.cc | 30 +++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 74e68fe12..bd5786857 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -437,7 +437,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) - info.files.insert(CanonPath(path)); + info.files.emplace(CanonPath(path), + statusFlags == GIT_STATUS_CURRENT + ? WorkdirInfo::State::Clean + : WorkdirInfo::State::Dirty); + else + info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; return 0; @@ -1202,6 +1207,15 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore } } +template +std::set getKeys(const std::map & c) +{ + std::set res; + for (auto & i : c) + res.insert(i.first); + return res; +} + ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); @@ -1214,7 +1228,7 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export ? makeEmptySourceAccessor() : AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set { wd.files }, + std::set { getKeys(wd.files) }, std::move(makeNotAllowedError)).cast(); if (exportIgnore) return make_ref(self, fileAccessor, std::nullopt); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index f45b5a504..12cee5db1 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -55,9 +55,14 @@ struct GitRepo in the repo yet. */ std::optional headRev; + enum State { Clean, Dirty }; + /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ - std::set files; + std::map files; + + /* The deleted files. */ + std::set deletedFiles; /* The submodules listed in .gitmodules of this workdir. */ std::vector submodules; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a6883a2d3..eec134980 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -685,7 +685,7 @@ struct GitInputScheme : InputScheme if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) - repoInfo.workdirInfo.files.insert(submodule.path); + repoInfo.workdirInfo.files.emplace(submodule.path, GitRepo::WorkdirInfo::State::Clean); auto repo = GitRepo::openRepo(repoInfo.url, false, false); @@ -793,10 +793,34 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { + auto makeFingerprint = [&](const Hash & rev) + { + return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); + }; + if (auto rev = input.getRev()) - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); - else + return makeFingerprint(*rev); + else { + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev) { + /* Calculate a fingerprint that takes into account the + deleted and modified/added files. */ + HashSink hashSink{HashAlgorithm::SHA512}; + for (auto & file : repoInfo.workdirInfo.files) + if (file.second == GitRepo::WorkdirInfo::State::Dirty) { + writeString("modified:", hashSink); + writeString(file.first.abs(), hashSink); + readFile(std::filesystem::path(repoInfo.url) + file.first.abs(), hashSink); + } + for (auto & file : repoInfo.workdirInfo.deletedFiles) { + writeString("deleted:", hashSink); + writeString(file.abs(), hashSink); + } + return makeFingerprint(*repoInfo.workdirInfo.headRev) + + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); + } return std::nullopt; + } } bool isLocked(const Input & input) const override From d044a05197c1d4066ef0c0b67ff0461ee3d5fa6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Dec 2024 15:50:39 +0100 Subject: [PATCH 151/243] Don't fingerprint dirty repos with submodules for now Fixes nixpkgsLibTests. --- src/libfetchers/git.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index eec134980..d47e731f1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "json-utils.hh" +#include "archive.hh" #include #include @@ -802,7 +803,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal && repoInfo.workdirInfo.headRev) { + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -810,7 +811,7 @@ struct GitInputScheme : InputScheme if (file.second == GitRepo::WorkdirInfo::State::Dirty) { writeString("modified:", hashSink); writeString(file.first.abs(), hashSink); - readFile(std::filesystem::path(repoInfo.url) + file.first.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.first.abs(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); From f469bc2ae4a221d0397775b2663ba90a2dff0b9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 13:17:31 +0100 Subject: [PATCH 152/243] Cache result of Input::getFingerprint() The fingerprint calculation can be expensive (especially for dirty Git trees) so we need to cache it. --- src/libfetchers/fetchers.cc | 14 +++++++++++--- src/libfetchers/fetchers.hh | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e15a460d0..b105c252a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -113,7 +113,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) std::optional Input::getFingerprint(ref store) const { - return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; + if (!scheme) return std::nullopt; + + if (cachedFingerprint) return *cachedFingerprint; + + auto fingerprint = scheme->getFingerprint(store, *this); + + cachedFingerprint = fingerprint; + + return fingerprint; } ParsedURL Input::toURL() const @@ -307,7 +315,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto accessor = makeStorePathAccessor(store, storePath); - accessor->fingerprint = scheme->getFingerprint(store, *this); + accessor->fingerprint = getFingerprint(store); return {accessor, *this}; } catch (Error & e) { @@ -318,7 +326,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); assert(!accessor->fingerprint); - accessor->fingerprint = scheme->getFingerprint(store, result); + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ff04d5551..841a44041 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -46,6 +46,11 @@ struct Input */ std::optional parent; + /** + * Cached result of getFingerprint(). + */ + mutable std::optional> cachedFingerprint; + public: /** * Create an `Input` from a URL. From 7ba933e989b9baf56c2b542c7788b38f6d9ccb50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 13:56:03 +0100 Subject: [PATCH 153/243] Cache calls to GitRepo::getWorkdirInfo() A command like `nix flake metadata` was causing about 4 calls to getWorkdirInfo(), which is slow for large repos (even when they're not dirty). --- src/libfetchers/git-utils.cc | 14 ++++++++++++++ src/libfetchers/git-utils.hh | 2 ++ src/libfetchers/git.cc | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index bd5786857..15fa540b3 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -5,6 +5,7 @@ #include "signals.hh" #include "users.hh" #include "fs-sink.hh" +#include "sync.hh" #include #include @@ -1276,4 +1277,17 @@ ref getTarballCache() return GitRepo::openRepo(repoDir, true, true); } +GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) +{ + static Sync> _cache; + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo(); + _cache.lock()->emplace(path, workdirInfo); + return workdirInfo; +} + } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 12cee5db1..2db64e438 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -70,6 +70,8 @@ struct GitRepo virtual WorkdirInfo getWorkdirInfo() = 0; + static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d47e731f1..a584542d1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -431,7 +431,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); return repoInfo; } From b9f60faab50f79c9d5fd4d11d1bf06feeb62edc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 14:51:41 +0100 Subject: [PATCH 154/243] Fix macOS build --- src/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 15fa540b3..a41546c44 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1279,7 +1279,7 @@ ref getTarballCache() GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + static Sync> _cache; { auto cache(_cache.lock()); auto i = cache->find(path); From 33852ead6b01b58cd66a474b6615595244dc56d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Dec 2024 15:31:19 +0100 Subject: [PATCH 155/243] Optimisation --- src/libfetchers/git-utils.cc | 21 ++++++--------------- src/libfetchers/git-utils.hh | 7 ++++--- src/libfetchers/git.cc | 13 ++++++------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index a41546c44..b54416b10 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -438,11 +438,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) - info.files.emplace(CanonPath(path), - statusFlags == GIT_STATUS_CURRENT - ? WorkdirInfo::State::Clean - : WorkdirInfo::State::Dirty); - else + { + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.dirtyFiles.insert(CanonPath(path)); + } else info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; @@ -1208,15 +1208,6 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore } } -template -std::set getKeys(const std::map & c) -{ - std::set res; - for (auto & i : c) - res.insert(i.first); - return res; -} - ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); @@ -1229,7 +1220,7 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export ? makeEmptySourceAccessor() : AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set { getKeys(wd.files) }, + std::set { wd.files }, std::move(makeNotAllowedError)).cast(); if (exportIgnore) return make_ref(self, fileAccessor, std::nullopt); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 2db64e438..ff115143f 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -55,11 +55,12 @@ struct GitRepo in the repo yet. */ std::optional headRev; - enum State { Clean, Dirty }; - /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ - std::map files; + std::set files; + + /* All modified or added files. */ + std::set dirtyFiles; /* The deleted files. */ std::set deletedFiles; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a584542d1..c73f53765 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -686,7 +686,7 @@ struct GitInputScheme : InputScheme if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) - repoInfo.workdirInfo.files.emplace(submodule.path, GitRepo::WorkdirInfo::State::Clean); + repoInfo.workdirInfo.files.insert(submodule.path); auto repo = GitRepo::openRepo(repoInfo.url, false, false); @@ -807,12 +807,11 @@ struct GitInputScheme : InputScheme /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; - for (auto & file : repoInfo.workdirInfo.files) - if (file.second == GitRepo::WorkdirInfo::State::Dirty) { - writeString("modified:", hashSink); - writeString(file.first.abs(), hashSink); - dumpPath(repoInfo.url + "/" + file.first.abs(), hashSink); - } + for (auto & file : repoInfo.workdirInfo.dirtyFiles) { + writeString("modified:", hashSink); + writeString(file.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); writeString(file.abs(), hashSink); From 757ea706449925636a99dfbcb09a09c62b1de319 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Dec 2024 13:13:39 +0100 Subject: [PATCH 156/243] Add a test --- tests/functional/flakes/flakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index de37ae1b7..6c466a0c7 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision) echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) From b33814093100ca15a2afdc1be207fcd4d1328a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:49:43 +0100 Subject: [PATCH 157/243] fix missing includes in various headers --- src/libexpr/lexer-helpers.hh | 8 ++++++++ src/libexpr/print-options.hh | 1 + src/libstore/path-regex.hh | 2 ++ src/libutil/callback.hh | 1 + src/libutil/config-impl.hh | 1 + src/libutil/regex-combinators.hh | 1 + src/libutil/unix/signals-impl.hh | 1 + src/nix/self-exe.hh | 2 ++ 8 files changed, 17 insertions(+) diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index caba6e18f..b4a166f46 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -1,5 +1,13 @@ #pragma once +#include + +// inluding the generated headers twice leads to errors +#ifndef BISON_HEADER +#include "lexer-tab.hh" +#include "parser-tab.hh" +#endif + namespace nix::lexer::internal { void initLoc(YYLTYPE * loc); diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index 080ba26b8..9ad54e532 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -5,6 +5,7 @@ */ #include +#include namespace nix { diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 56c2cfc1d..e34a305c5 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index 26c386d80..c2cada2f6 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index c3aa61ddb..94c2cb2e4 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -13,6 +13,7 @@ */ #include "config.hh" +#include "args.hh" namespace nix { diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa678..8a7b2c2c0 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -2,6 +2,7 @@ ///@file #include +#include namespace nix::regex { diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 2193922be..037416e7d 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -14,6 +14,7 @@ #include "error.hh" #include "logging.hh" #include "ansicolor.hh" +#include "signals.hh" #include #include diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh index 3161553ec..91e260f0b 100644 --- a/src/nix/self-exe.hh +++ b/src/nix/self-exe.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix { From b9bbdbeb0b5ec0bb3231d1e04d1825fdc1f6c4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 11:50:03 +0100 Subject: [PATCH 158/243] ca-fd-leak: add missing c casts --- tests/nixos/ca-fd-leak/sender.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c index 75e54fc8f..8356b2479 100644 --- a/tests/nixos/ca-fd-leak/sender.c +++ b/tests/nixos/ca-fd-leak/sender.c @@ -49,8 +49,8 @@ int main(int argc, char **argv) { msg.msg_controllen = CMSG_SPACE(sizeof(int)); // Write a single null byte too. - msg.msg_iov = malloc(sizeof(struct iovec)); - msg.msg_iov[0].iov_base = ""; + msg.msg_iov = (struct iovec*) malloc(sizeof(struct iovec)); + msg.msg_iov[0].iov_base = (void*) ""; msg.msg_iov[0].iov_len = 1; msg.msg_iovlen = 1; From eda331e53fc6322922f467263ded0b69c28dd427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:43:28 +0100 Subject: [PATCH 159/243] libutil/windows: only define headers/code if we are actually on windows all those includes otherwise break linters. --- src/libexpr/lexer-helpers.hh | 4 ++-- src/libstore/windows/pathlocks.cc | 11 +++++++---- src/libutil/windows/environment-variables.cc | 4 +++- src/libutil/windows/file-descriptor.cc | 2 ++ src/libutil/windows/file-system.cc | 2 ++ src/libutil/windows/muxable-pipe.cc | 12 +++++++----- src/libutil/windows/os-string.cc | 4 ++++ src/libutil/windows/processes.cc | 4 ++++ src/libutil/windows/users.cc | 2 ++ src/libutil/windows/windows-async-pipe.cc | 4 ++++ src/libutil/windows/windows-async-pipe.hh | 2 ++ src/libutil/windows/windows-error.cc | 2 ++ src/libutil/windows/windows-error.hh | 2 ++ 13 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index b4a166f46..d40f7b874 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -4,8 +4,8 @@ // inluding the generated headers twice leads to errors #ifndef BISON_HEADER -#include "lexer-tab.hh" -#include "parser-tab.hh" +# include "lexer-tab.hh" +# include "parser-tab.hh" #endif namespace nix::lexer::internal { diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 00761a8c3..197f5a1c4 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -2,10 +2,12 @@ #include "pathlocks.hh" #include "signals.hh" #include "util.hh" -#include -#include -#include -#include "windows-error.hh" + +#ifdef WIN32 +# include +# include +# include +# include "windows-error.hh" namespace nix { @@ -154,3 +156,4 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } +#endif diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 5ce8a1395..308a432e4 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" -#include "processenv.h" +#ifdef WIN32 +# include "processenv.h" namespace nix { @@ -43,3 +44,4 @@ int setEnvOs(const OsString & name, const OsString & value) } } +#endif diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 7b8a712e8..71f53ccb8 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,6 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" +#ifdef WIN32 #include #include #include @@ -152,3 +153,4 @@ Path windows::handleToPath(HANDLE handle) { #endif } +#endif diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index b15355efe..53271cef3 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,5 +1,6 @@ #include "file-system.hh" +#ifdef WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) @@ -15,3 +16,4 @@ Descriptor openDirectory(const std::filesystem::path & path) } } +#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 91a321f7c..af7e987e9 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,10 @@ -#include -#include "windows-error.hh" +#ifdef WIN32 +# include +# include "windows-error.hh" -#include "logging.hh" -#include "util.hh" -#include "muxable-pipe.hh" +# include "logging.hh" +# include "util.hh" +# include "muxable-pipe.hh" namespace nix { @@ -68,3 +69,4 @@ void MuxablePipePollState::iterate( } } +#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 7507f9030..26ad9cba0 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,6 +7,8 @@ #include "file-path-impl.hh" #include "util.hh" +#ifdef WIN32 + namespace nix { std::string os_string_to_string(PathViewNG::string_view path) @@ -22,3 +24,5 @@ std::filesystem::path::string_type string_to_os_string(std::string_view s) } } + +#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 7f34c5632..e69f1ed45 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,6 +23,8 @@ #include #include +#ifdef WIN32 + #define WIN32_LEAN_AND_MEAN #include @@ -386,3 +388,5 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } + +#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index db6c42df3..2780e45f4 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" +#ifdef WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -50,3 +51,4 @@ bool isRootUser() { } } +#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 4fa57ca36..13b563510 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,6 +1,8 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" +#ifdef WIN32 + namespace nix::windows { void AsyncPipe::createAsyncPipe(HANDLE iocp) @@ -47,3 +49,5 @@ void AsyncPipe::close() } } + +#endif diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 8f554e403..277336ed7 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,6 +2,7 @@ ///@file #include "file-descriptor.hh" +#ifdef WIN32 namespace nix::windows { @@ -25,3 +26,4 @@ public: }; } +#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index aead4af23..4cf4274da 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,5 +1,6 @@ #include "windows-error.hh" +#ifdef WIN32 #include #define WIN32_LEAN_AND_MEAN #include @@ -29,3 +30,4 @@ std::string WinError::renderError(DWORD lastError) } } +#endif diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 624b4c4cb..4e48ee859 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#ifdef WIN32 #include #include "error.hh" @@ -49,3 +50,4 @@ private: }; } +#endif From 3392a96901bfde94142fbfffd9d7d12549522ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:07:35 +0000 Subject: [PATCH 160/243] ca-fd-leak: fix various unsafe c handling --- tests/nixos/ca-fd-leak/smuggler.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/nixos/ca-fd-leak/smuggler.c b/tests/nixos/ca-fd-leak/smuggler.c index 82acf37e6..3f89af5bb 100644 --- a/tests/nixos/ca-fd-leak/smuggler.c +++ b/tests/nixos/ca-fd-leak/smuggler.c @@ -16,7 +16,7 @@ int main(int argc, char **argv) { struct sockaddr_un data; data.sun_family = AF_UNIX; data.sun_path[0] = 0; - strcpy(data.sun_path + 1, argv[1]); + strncpy(data.sun_path + 1, argv[1], sizeof(data.sun_path) - 1); int res = bind(sock, (const struct sockaddr *)&data, offsetof(struct sockaddr_un, sun_path) + strlen(argv[1]) @@ -57,10 +57,11 @@ int main(int argc, char **argv) { // Wait for a second connection, which will tell us that the build is // done a = accept(sock, 0, 0); + if (a < 0) perror("accept"); fprintf(stderr, "%s\n", "Got a second connection, rewriting the file"); // Write a new content to the file if (ftruncate(smuggling_fd, 0)) perror("ftruncate"); - char * new_content = "Pwned\n"; + const char * new_content = "Pwned\n"; int written_bytes = write(smuggling_fd, new_content, strlen(new_content)); if (written_bytes != strlen(new_content)) perror("write"); } From 6848154b2e657e64a592b60f00b68db8d911b89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 16 Dec 2024 12:47:20 +0000 Subject: [PATCH 161/243] regex-combinators: fix linter error when adding string_view and string --- src/libutil/regex-combinators.hh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 8a7b2c2c0..75ccd4e6c 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -3,6 +3,7 @@ #include #include +#include namespace nix::regex { @@ -11,22 +12,23 @@ namespace nix::regex { static inline std::string either(std::string_view a, std::string_view b) { - return std::string { a } + "|" + b; + std::stringstream ss; + ss << a << "|" << b; + return ss.str(); } static inline std::string group(std::string_view a) { - return std::string { "(" } + a + ")"; -} - -static inline std::string many(std::string_view a) -{ - return std::string { "(?:" } + a + ")*"; + std::stringstream ss; + ss << "(" << a << ")"; + return ss.str(); } static inline std::string list(std::string_view a) { - return std::string { a } + many(group("," + a)); + std::stringstream ss; + ss << a << "(," << a << ")*"; + return ss.str(); } } From 5cd5391dd8055bc0876204b6b38274f8c821c751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 20:41:39 +0100 Subject: [PATCH 162/243] checks: re-enable perlBindings --- flake.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index a5e68609e..44a764dd7 100644 --- a/flake.nix +++ b/flake.nix @@ -201,11 +201,7 @@ # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable this. - #perlBindings = self.hydraJobs.perlBindings.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; } # Add "passthru" tests // flatMapAttrs ({ From a8e1b4757e3022dff433693eb8ed3e757dd317c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 19 Dec 2024 18:35:28 +0100 Subject: [PATCH 163/243] filesystem/deletePath: remove unnecessary quotes from error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paths are already quoted: error: … while fetching the input 'path:/nix/store/rs2s2ca7xs87v82aps54m1p3sqrfz6c8-source' error: chmod '"/nix/store/rs2s2ca7xs87v82aps54m1p3sqrfz6c8-source"': Read-only file system --- src/libutil/file-system.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 829700336..50e90bb57 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -384,7 +384,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); + throw SysError("getting status of %1%", path); } if (!S_ISDIR(st.st_mode)) { @@ -416,15 +416,15 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); + throw SysError("chmod %1%", path); } int fd = openat(parentfd, path.c_str(), O_RDONLY); if (fd == -1) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); AutoCloseDir dir(fdopendir(fd)); if (!dir) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -433,13 +433,13 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (childName == "." || childName == "..") continue; _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed); } - if (errno) throw SysError("reading directory '%1%'", path); + if (errno) throw SysError("reading directory %1%", path); } int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); + throw SysError("cannot unlink %1%", path); } #else // TODO implement From 535724fd7953e9b465af419d66536dc810e3e6c2 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 11:14:04 -0800 Subject: [PATCH 164/243] tests/nixos/s3-binary-cache-store: test that "object does not exist" error message is properly formatted --- tests/nixos/s3-binary-cache-store.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 6c51fcba5..45f234645 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -10,6 +10,7 @@ let env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + objectThatDoesNotExist = "s3://my-cache/foo-that-does-not-exist?endpoint=http://server:9000®ion=eu-west-1"; in { name = "s3-binary-cache-store"; @@ -54,6 +55,12 @@ in { # Test fetchurl on s3:// URLs while we're at it. client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") + # Test that the format string in the error message is properly setup and won't display `%s` instead of the failed URI + msg = client.fail("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"${objectThatDoesNotExist}\"; }' 2>&1") + if "S3 object '${objectThatDoesNotExist}' does not exist" not in msg: + print(msg) # So that you can see the message that was improperly formatted + raise Exception("Error message formatting didn't work") + # Copy a package from the binary cache. client.fail("nix path-info ${pkgA}") From f0c1262d2319ff6967139d8f8599ed6176ad1263 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 11:15:25 -0800 Subject: [PATCH 165/243] tests/nixos/s3-binary-cache-store: disable default substituter so it runs faster Since networking is disabled in these VMs, trying to talk to the default cache.nixos.org slows the test down (since it can't resolve it). --- tests/nixos/s3-binary-cache-store.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 45f234645..83b85c032 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -21,7 +21,10 @@ in { { virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgs.minio-client ]; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; services.minio = { enable = true; region = "eu-west-1"; @@ -36,7 +39,10 @@ in { client = { config, pkgs, ... }: { virtualisation.writableStore = true; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; }; }; From b978fa845022d448c11a1d2e9d0347a57edecbd8 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 10:16:47 -0800 Subject: [PATCH 166/243] libstore: fixup unformatted uri when S3 getObject fails --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 42b93cfe0..8439cc39c 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -767,7 +767,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); res.urls.push_back(request.uri); callback(std::move(res)); From 9223d64ac62385d3b75cfbf5bac491fc14d033ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Dec 2024 16:03:13 +0100 Subject: [PATCH 167/243] Remove dead code --- src/libflake/flake/flakeref.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 41e991c20..3a43a51f8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -195,10 +195,6 @@ std::pair parsePathFlakeRefWithFragment( throw BadURL("flake reference '%s' is not an absolute path", url); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - return fromParsedURL(fetchSettings, { .url = path, .base = path, From 4a91e627a7617cc3d654967358fd29df076ab853 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Wed, 25 Dec 2024 02:41:20 +0100 Subject: [PATCH 168/243] fix: ignore symlinks in fsync-store-paths Fixes: https://github.com/NixOS/nix/issues/12099 --- src/libutil/file-system.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 50e90bb57..793eb2d9f 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -331,7 +331,7 @@ void syncParent(const Path & path) void recursiveSync(const Path & path) { - /* If it's a file, just fsync and return. */ + /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); @@ -339,7 +339,8 @@ void recursiveSync(const Path & path) throw SysError("opening file '%1%'", path); fd.fsync(); return; - } + } else if (S_ISLNK(st.st_mode)) + return; /* Otherwise, perform a depth-first traversal of the directory and fsync all the files. */ From fe5f02c2c274e8f48a1072807fc9bb3f892584a4 Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:56:12 +0100 Subject: [PATCH 169/243] ci: lock Ubuntu runner to ubuntu-22.04 Lock the Ubuntu runner to ubuntu-22.04 to avoid accidental updates [1] and increase reproducibility. [1]: https://github.com/actions/runner-images/issues/10636 --- .github/workflows/ci.yml | 14 +++++++------- .github/workflows/labels.yml | 2 +- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index addafb9f8..2426ab166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: read-all jobs: eval: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-22.04, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -37,7 +37,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-22.04, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -68,7 +68,7 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' - run: brew install fish if: matrix.os == 'macos-latest' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" @@ -86,7 +86,7 @@ jobs: permissions: contents: none name: Check Docker secrets present for installer tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: docker: ${{ steps.secret.outputs.docker }} steps: @@ -106,7 +106,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && github.ref_name == 'master' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check for secrets id: secret diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 34aa4e6bd..9d2ac80a3 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -15,7 +15,7 @@ permissions: jobs: labels: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.repository_owner == 'NixOS' steps: - uses: actions/labeler@v5 diff --git a/.mergify.yml b/.mergify.yml index 70fccae49..c2e115487 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,9 @@ queue_rules: # all required tests need to go here merge_conditions: - check-success=tests (macos-latest) - - check-success=tests (ubuntu-latest) + - check-success=tests (ubuntu-22.04) - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) + - check-success=installer_test (ubuntu-22.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 30aa7d0d5..7e8762fe0 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-22.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From cf69c99f3ec16deb25ae7a57ee579431828da249 Mon Sep 17 00:00:00 2001 From: Parker Jones <49885263+knotapun@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:11:36 -0500 Subject: [PATCH 170/243] Make `readFileType` doc string consistent The primitive `readFileType p` has a list of acceptable types, and so does `readDir path` This edit makes the formatting of the list consistent between themselves, and other parts of the documentation. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c1f1cf4c1..f19dd473a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2045,7 +2045,7 @@ static RegisterPrimOp primop_readFileType({ .args = {"p"}, .doc = R"( Determine the directory entry type of a filesystem node, being - one of "directory", "regular", "symlink", or "unknown". + one of `"directory"`, `"regular"`, `"symlink"`, or `"unknown"`. )", .fun = prim_readFileType, }); From 92e30955b9a4acbc01cd6519552ada3385ef3c7e Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sun, 15 Dec 2024 02:29:56 +0100 Subject: [PATCH 171/243] try to calculate character width --- maintainers/flake-module.nix | 1 + src/libutil-tests/terminal.cc | 4 + src/libutil/meson.build | 6 + src/libutil/package.nix | 1 + src/libutil/terminal.cc | 90 +- src/libutil/widecharwidth/LICENSE | 4 + src/libutil/widecharwidth/widechar_width.h | 1559 ++++++++++++++++++++ 7 files changed, 1636 insertions(+), 29 deletions(-) create mode 100644 src/libutil/widecharwidth/LICENSE create mode 100644 src/libutil/widecharwidth/widechar_width.h diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1a134b91a..6b311c62e 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -356,6 +356,7 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/widecharwidth/widechar_width\.h$'' # vendored source ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 714d5a237..f4fc6e770 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -55,6 +55,10 @@ TEST(filterANSIEscapes, utf8) ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 6), "f🔍bar"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 3), "f🔍"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 2), "f"); + ASSERT_EQ(filterANSIEscapes("foo\u0301", true, 3), "foó"); } TEST(filterANSIEscapes, osc8) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bbe7872cf..2c3e3a954 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -108,6 +108,8 @@ deps_private += cpuid nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json +cxx = meson.get_compiler('cpp') + config_h = configure_file( configuration : configdata, output : 'config-util.hh', @@ -168,6 +170,10 @@ sources = files( ) include_dirs = [include_directories('.')] +if not cxx.has_header('widechar_width.h', required : false) + # use vendored widechar_width.h + include_dirs += include_directories('./widecharwidth') +endif headers = [config_h] + files( 'abstract-setting-to-json.hh', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 69ebbf726..0a9f65782 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -29,6 +29,7 @@ mkMesonLibrary (finalAttrs: { ./nix-meson-build-support ../../.version ./.version + ./widecharwidth ./meson.build ./meson.options ./linux/meson.build diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 4c127ddb0..8a8373f1b 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -11,6 +11,53 @@ # include #endif #include +#include + +namespace { + +inline std::pair charWidthUTF8Helper(std::string_view s) +{ + size_t bytes = 1; + uint32_t ch = s[0]; + uint32_t max = 1U << 7; + if ((ch & 0x80U) == 0U) { + } else if ((ch & 0xe0U) == 0xc0U) { + ch &= 0x1fU; + bytes = 2; + max = 1U << 11; + } else if ((ch & 0xf0U) == 0xe0U) { + ch &= 0x0fU; + bytes = 3; + max = 1U << 16; + } else if ((ch & 0xf8U) == 0xf0U) { + ch &= 0x07U; + bytes = 4; + max = 0x110000U; + } else { + return {bytes, bytes}; // invalid UTF-8 start byte + } + for (size_t i = 1; i < bytes; i++) { + if (i < s.size() && (s[i] & 0xc0) == 0x80) { + ch = (ch << 6) | (s[i] & 0x3f); + } else { + return {i, i}; // invalid UTF-8 encoding; assume one character per byte + } + } + int width = bytes; // in case of overlong encoding + if (ch < max) { + width = widechar_wcwidth(ch); + if (width == widechar_ambiguous) { + width = 1; // just a guess... + } else if (width == widechar_widened_in_9) { + width = 2; + } else if (width < 0) { + width = 0; + } + } + return {width, bytes}; +} + +} namespace nix { @@ -30,7 +77,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w size_t w = 0; auto i = s.begin(); - while (w < (size_t) width && i != s.end()) { + while (i != s.end()) { if (*i == '\e') { std::string e; @@ -61,10 +108,12 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w } else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } + do { + if (++w > (size_t) width) + return t; + t += ' '; + } while (w % 8); + i++; } else if (*i == '\r' || *i == '\a') @@ -72,35 +121,18 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w i++; else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; + auto [chWidth, bytes] = charWidthUTF8Helper({i, s.end()}); + w += chWidth; + if (w > (size_t) width) { + break; + } + t += {i, i + bytes}; + i += bytes; } } - return t; } - ////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/widecharwidth/LICENSE b/src/libutil/widecharwidth/LICENSE new file mode 100644 index 000000000..d3b1dd767 --- /dev/null +++ b/src/libutil/widecharwidth/LICENSE @@ -0,0 +1,4 @@ +widecharwidth - wcwidth implementation +Written in 2018 by ridiculous_fish +To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain worldwide. This software is distributed without any warranty. +You should have received a copy of the CC0 Public Domain Dedication along with this software. If not, see . diff --git a/src/libutil/widecharwidth/widechar_width.h b/src/libutil/widecharwidth/widechar_width.h new file mode 100644 index 000000000..92e63e913 --- /dev/null +++ b/src/libutil/widecharwidth/widechar_width.h @@ -0,0 +1,1559 @@ +/** + * widechar_width.h for Unicode 16.0.0 + * See https://github.com/ridiculousfish/widecharwidth/ + * + * SHA1 file hashes: + * ( + * the hashes for generate.py and the template are git object hashes, + * use `git log --all --find-object=` in the widecharwidth repository + * to see which commit they correspond to, + * or run `git hash-object` on the file to compare. + * The other hashes are simple `sha1sum` style hashes. + * ) + * + * generate.py: 2747bb9402d8eeeca8e566ff947f14308511ecb1 + * template.js: 1249763c5b7c1e308aeb4ca64f1e15bce1fab9b3 + * UnicodeData.txt: 91df83276154240bcedef82a09bde77aa182cf8d + * EastAsianWidth.txt: 0885c0fc1c21eb58954a3bfb785d78559b361d92 + * emoji-data.txt: 1df2f8329dd9f5c238674807de736f316c6b9d87 + */ + +#ifndef WIDECHAR_WIDTH_H +#define WIDECHAR_WIDTH_H + +#include +#include +#include +#include + +namespace { + +/* Special width values */ +enum { + widechar_nonprint = -1, // The character is not printable. + widechar_combining = -2, // The character is a zero-width combiner. + widechar_ambiguous = -3, // The character is East-Asian ambiguous width. + widechar_private_use = -4, // The character is for private use. + widechar_unassigned = -5, // The character is unassigned. + widechar_widened_in_9 = -6, // Width is 1 in Unicode 8, 2 in Unicode 9+. + widechar_non_character = -7 // The character is a noncharacter. +}; + +/* An inclusive range of characters. */ +struct widechar_range { + uint32_t lo; + uint32_t hi; +}; + +/* Simple ASCII characters - used a lot, so we check them first. */ +static const struct widechar_range widechar_ascii_table[] = { + {0x00020, 0x0007E} +}; + +/* Private usage range. */ +static const struct widechar_range widechar_private_table[] = { + {0x0E000, 0x0F8FF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Nonprinting characters. */ +static const struct widechar_range widechar_nonprint_table[] = { + {0x00000, 0x0001F}, + {0x0007F, 0x0009F}, + {0x000AD, 0x000AD}, + {0x00600, 0x00605}, + {0x0061C, 0x0061C}, + {0x006DD, 0x006DD}, + {0x0070F, 0x0070F}, + {0x00890, 0x00891}, + {0x008E2, 0x008E2}, + {0x0180E, 0x0180E}, + {0x0200B, 0x0200F}, + {0x02028, 0x0202E}, + {0x02060, 0x02064}, + {0x02066, 0x0206F}, + {0x0D800, 0x0DFFF}, + {0x0FEFF, 0x0FEFF}, + {0x0FFF9, 0x0FFFB}, + {0x110BD, 0x110BD}, + {0x110CD, 0x110CD}, + {0x13430, 0x1343F}, + {0x1BCA0, 0x1BCA3}, + {0x1D173, 0x1D17A}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F} +}; + +/* Width 0 combining marks. */ +static const struct widechar_range widechar_combining_table[] = { + {0x00300, 0x0036F}, + {0x00483, 0x00489}, + {0x00591, 0x005BD}, + {0x005BF, 0x005BF}, + {0x005C1, 0x005C2}, + {0x005C4, 0x005C5}, + {0x005C7, 0x005C7}, + {0x00610, 0x0061A}, + {0x0064B, 0x0065F}, + {0x00670, 0x00670}, + {0x006D6, 0x006DC}, + {0x006DF, 0x006E4}, + {0x006E7, 0x006E8}, + {0x006EA, 0x006ED}, + {0x00711, 0x00711}, + {0x00730, 0x0074A}, + {0x007A6, 0x007B0}, + {0x007EB, 0x007F3}, + {0x007FD, 0x007FD}, + {0x00816, 0x00819}, + {0x0081B, 0x00823}, + {0x00825, 0x00827}, + {0x00829, 0x0082D}, + {0x00859, 0x0085B}, + {0x00897, 0x0089F}, + {0x008CA, 0x008E1}, + {0x008E3, 0x00903}, + {0x0093A, 0x0093C}, + {0x0093E, 0x0094F}, + {0x00951, 0x00957}, + {0x00962, 0x00963}, + {0x00981, 0x00983}, + {0x009BC, 0x009BC}, + {0x009BE, 0x009C4}, + {0x009C7, 0x009C8}, + {0x009CB, 0x009CD}, + {0x009D7, 0x009D7}, + {0x009E2, 0x009E3}, + {0x009FE, 0x009FE}, + {0x00A01, 0x00A03}, + {0x00A3C, 0x00A3C}, + {0x00A3E, 0x00A42}, + {0x00A47, 0x00A48}, + {0x00A4B, 0x00A4D}, + {0x00A51, 0x00A51}, + {0x00A70, 0x00A71}, + {0x00A75, 0x00A75}, + {0x00A81, 0x00A83}, + {0x00ABC, 0x00ABC}, + {0x00ABE, 0x00AC5}, + {0x00AC7, 0x00AC9}, + {0x00ACB, 0x00ACD}, + {0x00AE2, 0x00AE3}, + {0x00AFA, 0x00AFF}, + {0x00B01, 0x00B03}, + {0x00B3C, 0x00B3C}, + {0x00B3E, 0x00B44}, + {0x00B47, 0x00B48}, + {0x00B4B, 0x00B4D}, + {0x00B55, 0x00B57}, + {0x00B62, 0x00B63}, + {0x00B82, 0x00B82}, + {0x00BBE, 0x00BC2}, + {0x00BC6, 0x00BC8}, + {0x00BCA, 0x00BCD}, + {0x00BD7, 0x00BD7}, + {0x00C00, 0x00C04}, + {0x00C3C, 0x00C3C}, + {0x00C3E, 0x00C44}, + {0x00C46, 0x00C48}, + {0x00C4A, 0x00C4D}, + {0x00C55, 0x00C56}, + {0x00C62, 0x00C63}, + {0x00C81, 0x00C83}, + {0x00CBC, 0x00CBC}, + {0x00CBE, 0x00CC4}, + {0x00CC6, 0x00CC8}, + {0x00CCA, 0x00CCD}, + {0x00CD5, 0x00CD6}, + {0x00CE2, 0x00CE3}, + {0x00CF3, 0x00CF3}, + {0x00D00, 0x00D03}, + {0x00D3B, 0x00D3C}, + {0x00D3E, 0x00D44}, + {0x00D46, 0x00D48}, + {0x00D4A, 0x00D4D}, + {0x00D57, 0x00D57}, + {0x00D62, 0x00D63}, + {0x00D81, 0x00D83}, + {0x00DCA, 0x00DCA}, + {0x00DCF, 0x00DD4}, + {0x00DD6, 0x00DD6}, + {0x00DD8, 0x00DDF}, + {0x00DF2, 0x00DF3}, + {0x00E31, 0x00E31}, + {0x00E34, 0x00E3A}, + {0x00E47, 0x00E4E}, + {0x00EB1, 0x00EB1}, + {0x00EB4, 0x00EBC}, + {0x00EC8, 0x00ECE}, + {0x00F18, 0x00F19}, + {0x00F35, 0x00F35}, + {0x00F37, 0x00F37}, + {0x00F39, 0x00F39}, + {0x00F3E, 0x00F3F}, + {0x00F71, 0x00F84}, + {0x00F86, 0x00F87}, + {0x00F8D, 0x00F97}, + {0x00F99, 0x00FBC}, + {0x00FC6, 0x00FC6}, + {0x0102B, 0x0103E}, + {0x01056, 0x01059}, + {0x0105E, 0x01060}, + {0x01062, 0x01064}, + {0x01067, 0x0106D}, + {0x01071, 0x01074}, + {0x01082, 0x0108D}, + {0x0108F, 0x0108F}, + {0x0109A, 0x0109D}, + {0x0135D, 0x0135F}, + {0x01712, 0x01715}, + {0x01732, 0x01734}, + {0x01752, 0x01753}, + {0x01772, 0x01773}, + {0x017B4, 0x017D3}, + {0x017DD, 0x017DD}, + {0x0180B, 0x0180D}, + {0x0180F, 0x0180F}, + {0x01885, 0x01886}, + {0x018A9, 0x018A9}, + {0x01920, 0x0192B}, + {0x01930, 0x0193B}, + {0x01A17, 0x01A1B}, + {0x01A55, 0x01A5E}, + {0x01A60, 0x01A7C}, + {0x01A7F, 0x01A7F}, + {0x01AB0, 0x01ACE}, + {0x01B00, 0x01B04}, + {0x01B34, 0x01B44}, + {0x01B6B, 0x01B73}, + {0x01B80, 0x01B82}, + {0x01BA1, 0x01BAD}, + {0x01BE6, 0x01BF3}, + {0x01C24, 0x01C37}, + {0x01CD0, 0x01CD2}, + {0x01CD4, 0x01CE8}, + {0x01CED, 0x01CED}, + {0x01CF4, 0x01CF4}, + {0x01CF7, 0x01CF9}, + {0x01DC0, 0x01DFF}, + {0x020D0, 0x020F0}, + {0x02CEF, 0x02CF1}, + {0x02D7F, 0x02D7F}, + {0x02DE0, 0x02DFF}, + {0x0302A, 0x0302F}, + {0x03099, 0x0309A}, + {0x0A66F, 0x0A672}, + {0x0A674, 0x0A67D}, + {0x0A69E, 0x0A69F}, + {0x0A6F0, 0x0A6F1}, + {0x0A802, 0x0A802}, + {0x0A806, 0x0A806}, + {0x0A80B, 0x0A80B}, + {0x0A823, 0x0A827}, + {0x0A82C, 0x0A82C}, + {0x0A880, 0x0A881}, + {0x0A8B4, 0x0A8C5}, + {0x0A8E0, 0x0A8F1}, + {0x0A8FF, 0x0A8FF}, + {0x0A926, 0x0A92D}, + {0x0A947, 0x0A953}, + {0x0A980, 0x0A983}, + {0x0A9B3, 0x0A9C0}, + {0x0A9E5, 0x0A9E5}, + {0x0AA29, 0x0AA36}, + {0x0AA43, 0x0AA43}, + {0x0AA4C, 0x0AA4D}, + {0x0AA7B, 0x0AA7D}, + {0x0AAB0, 0x0AAB0}, + {0x0AAB2, 0x0AAB4}, + {0x0AAB7, 0x0AAB8}, + {0x0AABE, 0x0AABF}, + {0x0AAC1, 0x0AAC1}, + {0x0AAEB, 0x0AAEF}, + {0x0AAF5, 0x0AAF6}, + {0x0ABE3, 0x0ABEA}, + {0x0ABEC, 0x0ABED}, + {0x0FB1E, 0x0FB1E}, + {0x0FE00, 0x0FE0F}, + {0x0FE20, 0x0FE2F}, + {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, + {0x10376, 0x1037A}, + {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, + {0x10A0C, 0x10A0F}, + {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, + {0x10AE5, 0x10AE6}, + {0x10D24, 0x10D27}, + {0x10D69, 0x10D6D}, + {0x10EAB, 0x10EAC}, + {0x10EFC, 0x10EFF}, + {0x10F46, 0x10F50}, + {0x10F82, 0x10F85}, + {0x11000, 0x11002}, + {0x11038, 0x11046}, + {0x11070, 0x11070}, + {0x11073, 0x11074}, + {0x1107F, 0x11082}, + {0x110B0, 0x110BA}, + {0x110C2, 0x110C2}, + {0x11100, 0x11102}, + {0x11127, 0x11134}, + {0x11145, 0x11146}, + {0x11173, 0x11173}, + {0x11180, 0x11182}, + {0x111B3, 0x111C0}, + {0x111C9, 0x111CC}, + {0x111CE, 0x111CF}, + {0x1122C, 0x11237}, + {0x1123E, 0x1123E}, + {0x11241, 0x11241}, + {0x112DF, 0x112EA}, + {0x11300, 0x11303}, + {0x1133B, 0x1133C}, + {0x1133E, 0x11344}, + {0x11347, 0x11348}, + {0x1134B, 0x1134D}, + {0x11357, 0x11357}, + {0x11362, 0x11363}, + {0x11366, 0x1136C}, + {0x11370, 0x11374}, + {0x113B8, 0x113C0}, + {0x113C2, 0x113C2}, + {0x113C5, 0x113C5}, + {0x113C7, 0x113CA}, + {0x113CC, 0x113D0}, + {0x113D2, 0x113D2}, + {0x113E1, 0x113E2}, + {0x11435, 0x11446}, + {0x1145E, 0x1145E}, + {0x114B0, 0x114C3}, + {0x115AF, 0x115B5}, + {0x115B8, 0x115C0}, + {0x115DC, 0x115DD}, + {0x11630, 0x11640}, + {0x116AB, 0x116B7}, + {0x1171D, 0x1172B}, + {0x1182C, 0x1183A}, + {0x11930, 0x11935}, + {0x11937, 0x11938}, + {0x1193B, 0x1193E}, + {0x11940, 0x11940}, + {0x11942, 0x11943}, + {0x119D1, 0x119D7}, + {0x119DA, 0x119E0}, + {0x119E4, 0x119E4}, + {0x11A01, 0x11A0A}, + {0x11A33, 0x11A39}, + {0x11A3B, 0x11A3E}, + {0x11A47, 0x11A47}, + {0x11A51, 0x11A5B}, + {0x11A8A, 0x11A99}, + {0x11C2F, 0x11C36}, + {0x11C38, 0x11C3F}, + {0x11C92, 0x11CA7}, + {0x11CA9, 0x11CB6}, + {0x11D31, 0x11D36}, + {0x11D3A, 0x11D3A}, + {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D45}, + {0x11D47, 0x11D47}, + {0x11D8A, 0x11D8E}, + {0x11D90, 0x11D91}, + {0x11D93, 0x11D97}, + {0x11EF3, 0x11EF6}, + {0x11F00, 0x11F01}, + {0x11F03, 0x11F03}, + {0x11F34, 0x11F3A}, + {0x11F3E, 0x11F42}, + {0x11F5A, 0x11F5A}, + {0x13440, 0x13440}, + {0x13447, 0x13455}, + {0x1611E, 0x1612F}, + {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, + {0x16F4F, 0x16F4F}, + {0x16F51, 0x16F87}, + {0x16F8F, 0x16F92}, + {0x16FE4, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x1BC9D, 0x1BC9E}, + {0x1CF00, 0x1CF2D}, + {0x1CF30, 0x1CF46}, + {0x1D165, 0x1D169}, + {0x1D16D, 0x1D172}, + {0x1D17B, 0x1D182}, + {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, + {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, + {0x1DAA1, 0x1DAAF}, + {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, + {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, + {0x1E08F, 0x1E08F}, + {0x1E130, 0x1E136}, + {0x1E2AE, 0x1E2AE}, + {0x1E2EC, 0x1E2EF}, + {0x1E4EC, 0x1E4EF}, + {0x1E5EE, 0x1E5EF}, + {0x1E8D0, 0x1E8D6}, + {0x1E944, 0x1E94A}, + {0xE0100, 0xE01EF} +}; + +/* Width 0 combining letters. */ +static const struct widechar_range widechar_combiningletters_table[] = { + {0x01160, 0x011FF}, + {0x0D7B0, 0x0D7FF} +}; + +/* Width 2 characters. */ +static const struct widechar_range widechar_doublewide_table[] = { + {0x01100, 0x0115F}, + {0x02329, 0x0232A}, + {0x02630, 0x02637}, + {0x0268A, 0x0268F}, + {0x02E80, 0x02E99}, + {0x02E9B, 0x02EF3}, + {0x02F00, 0x02FD5}, + {0x02FF0, 0x0303E}, + {0x03041, 0x03096}, + {0x03099, 0x030FF}, + {0x03105, 0x0312F}, + {0x03131, 0x0318E}, + {0x03190, 0x031E5}, + {0x031EF, 0x0321E}, + {0x03220, 0x03247}, + {0x03250, 0x0A48C}, + {0x0A490, 0x0A4C6}, + {0x0A960, 0x0A97C}, + {0x0AC00, 0x0D7A3}, + {0x0F900, 0x0FAFF}, + {0x0FE10, 0x0FE19}, + {0x0FE30, 0x0FE52}, + {0x0FE54, 0x0FE66}, + {0x0FE68, 0x0FE6B}, + {0x0FF01, 0x0FF60}, + {0x0FFE0, 0x0FFE6}, + {0x16FE0, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, + {0x18CFF, 0x18D08}, + {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, + {0x1B000, 0x1B122}, + {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, + {0x1B170, 0x1B2FB}, + {0x1D300, 0x1D356}, + {0x1D360, 0x1D376}, + {0x1F200, 0x1F200}, + {0x1F202, 0x1F202}, + {0x1F210, 0x1F219}, + {0x1F21B, 0x1F22E}, + {0x1F230, 0x1F231}, + {0x1F237, 0x1F237}, + {0x1F23B, 0x1F23B}, + {0x1F240, 0x1F248}, + {0x1F260, 0x1F265}, + {0x1F57A, 0x1F57A}, + {0x1F5A4, 0x1F5A4}, + {0x1F6D1, 0x1F6D2}, + {0x1F6D5, 0x1F6D7}, + {0x1F6DC, 0x1F6DF}, + {0x1F6F4, 0x1F6FC}, + {0x1F7E0, 0x1F7EB}, + {0x1F7F0, 0x1F7F0}, + {0x1F90C, 0x1F90F}, + {0x1F919, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F97F}, + {0x1F985, 0x1F9BF}, + {0x1F9C1, 0x1F9FF}, + {0x1FA70, 0x1FA7C}, + {0x1FA80, 0x1FA89}, + {0x1FA8F, 0x1FAC6}, + {0x1FACE, 0x1FADC}, + {0x1FADF, 0x1FAE9}, + {0x1FAF0, 0x1FAF8}, + {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD} +}; + +/* Ambiguous-width characters. */ +static const struct widechar_range widechar_ambiguous_table[] = { + {0x000A1, 0x000A1}, + {0x000A4, 0x000A4}, + {0x000A7, 0x000A8}, + {0x000AA, 0x000AA}, + {0x000AD, 0x000AE}, + {0x000B0, 0x000B4}, + {0x000B6, 0x000BA}, + {0x000BC, 0x000BF}, + {0x000C6, 0x000C6}, + {0x000D0, 0x000D0}, + {0x000D7, 0x000D8}, + {0x000DE, 0x000E1}, + {0x000E6, 0x000E6}, + {0x000E8, 0x000EA}, + {0x000EC, 0x000ED}, + {0x000F0, 0x000F0}, + {0x000F2, 0x000F3}, + {0x000F7, 0x000FA}, + {0x000FC, 0x000FC}, + {0x000FE, 0x000FE}, + {0x00101, 0x00101}, + {0x00111, 0x00111}, + {0x00113, 0x00113}, + {0x0011B, 0x0011B}, + {0x00126, 0x00127}, + {0x0012B, 0x0012B}, + {0x00131, 0x00133}, + {0x00138, 0x00138}, + {0x0013F, 0x00142}, + {0x00144, 0x00144}, + {0x00148, 0x0014B}, + {0x0014D, 0x0014D}, + {0x00152, 0x00153}, + {0x00166, 0x00167}, + {0x0016B, 0x0016B}, + {0x001CE, 0x001CE}, + {0x001D0, 0x001D0}, + {0x001D2, 0x001D2}, + {0x001D4, 0x001D4}, + {0x001D6, 0x001D6}, + {0x001D8, 0x001D8}, + {0x001DA, 0x001DA}, + {0x001DC, 0x001DC}, + {0x00251, 0x00251}, + {0x00261, 0x00261}, + {0x002C4, 0x002C4}, + {0x002C7, 0x002C7}, + {0x002C9, 0x002CB}, + {0x002CD, 0x002CD}, + {0x002D0, 0x002D0}, + {0x002D8, 0x002DB}, + {0x002DD, 0x002DD}, + {0x002DF, 0x002DF}, + {0x00300, 0x0036F}, + {0x00391, 0x003A1}, + {0x003A3, 0x003A9}, + {0x003B1, 0x003C1}, + {0x003C3, 0x003C9}, + {0x00401, 0x00401}, + {0x00410, 0x0044F}, + {0x00451, 0x00451}, + {0x02010, 0x02010}, + {0x02013, 0x02016}, + {0x02018, 0x02019}, + {0x0201C, 0x0201D}, + {0x02020, 0x02022}, + {0x02024, 0x02027}, + {0x02030, 0x02030}, + {0x02032, 0x02033}, + {0x02035, 0x02035}, + {0x0203B, 0x0203B}, + {0x0203E, 0x0203E}, + {0x02074, 0x02074}, + {0x0207F, 0x0207F}, + {0x02081, 0x02084}, + {0x020AC, 0x020AC}, + {0x02103, 0x02103}, + {0x02105, 0x02105}, + {0x02109, 0x02109}, + {0x02113, 0x02113}, + {0x02116, 0x02116}, + {0x02121, 0x02122}, + {0x02126, 0x02126}, + {0x0212B, 0x0212B}, + {0x02153, 0x02154}, + {0x0215B, 0x0215E}, + {0x02160, 0x0216B}, + {0x02170, 0x02179}, + {0x02189, 0x02189}, + {0x02190, 0x02199}, + {0x021B8, 0x021B9}, + {0x021D2, 0x021D2}, + {0x021D4, 0x021D4}, + {0x021E7, 0x021E7}, + {0x02200, 0x02200}, + {0x02202, 0x02203}, + {0x02207, 0x02208}, + {0x0220B, 0x0220B}, + {0x0220F, 0x0220F}, + {0x02211, 0x02211}, + {0x02215, 0x02215}, + {0x0221A, 0x0221A}, + {0x0221D, 0x02220}, + {0x02223, 0x02223}, + {0x02225, 0x02225}, + {0x02227, 0x0222C}, + {0x0222E, 0x0222E}, + {0x02234, 0x02237}, + {0x0223C, 0x0223D}, + {0x02248, 0x02248}, + {0x0224C, 0x0224C}, + {0x02252, 0x02252}, + {0x02260, 0x02261}, + {0x02264, 0x02267}, + {0x0226A, 0x0226B}, + {0x0226E, 0x0226F}, + {0x02282, 0x02283}, + {0x02286, 0x02287}, + {0x02295, 0x02295}, + {0x02299, 0x02299}, + {0x022A5, 0x022A5}, + {0x022BF, 0x022BF}, + {0x02312, 0x02312}, + {0x02460, 0x024E9}, + {0x024EB, 0x0254B}, + {0x02550, 0x02573}, + {0x02580, 0x0258F}, + {0x02592, 0x02595}, + {0x025A0, 0x025A1}, + {0x025A3, 0x025A9}, + {0x025B2, 0x025B3}, + {0x025B6, 0x025B7}, + {0x025BC, 0x025BD}, + {0x025C0, 0x025C1}, + {0x025C6, 0x025C8}, + {0x025CB, 0x025CB}, + {0x025CE, 0x025D1}, + {0x025E2, 0x025E5}, + {0x025EF, 0x025EF}, + {0x02605, 0x02606}, + {0x02609, 0x02609}, + {0x0260E, 0x0260F}, + {0x0261C, 0x0261C}, + {0x0261E, 0x0261E}, + {0x02640, 0x02640}, + {0x02642, 0x02642}, + {0x02660, 0x02661}, + {0x02663, 0x02665}, + {0x02667, 0x0266A}, + {0x0266C, 0x0266D}, + {0x0266F, 0x0266F}, + {0x0269E, 0x0269F}, + {0x026BF, 0x026BF}, + {0x026C6, 0x026CD}, + {0x026CF, 0x026D3}, + {0x026D5, 0x026E1}, + {0x026E3, 0x026E3}, + {0x026E8, 0x026E9}, + {0x026EB, 0x026F1}, + {0x026F4, 0x026F4}, + {0x026F6, 0x026F9}, + {0x026FB, 0x026FC}, + {0x026FE, 0x026FF}, + {0x0273D, 0x0273D}, + {0x02776, 0x0277F}, + {0x02B56, 0x02B59}, + {0x03248, 0x0324F}, + {0x0E000, 0x0F8FF}, + {0x0FE00, 0x0FE0F}, + {0x0FFFD, 0x0FFFD}, + {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, + {0x1F130, 0x1F169}, + {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, + {0x1F19B, 0x1F1AC}, + {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Unassigned characters. */ +static const struct widechar_range widechar_unassigned_table[] = { + {0x00378, 0x00379}, + {0x00380, 0x00383}, + {0x0038B, 0x0038B}, + {0x0038D, 0x0038D}, + {0x003A2, 0x003A2}, + {0x00530, 0x00530}, + {0x00557, 0x00558}, + {0x0058B, 0x0058C}, + {0x00590, 0x00590}, + {0x005C8, 0x005CF}, + {0x005EB, 0x005EE}, + {0x005F5, 0x005FF}, + {0x0070E, 0x0070E}, + {0x0074B, 0x0074C}, + {0x007B2, 0x007BF}, + {0x007FB, 0x007FC}, + {0x0082E, 0x0082F}, + {0x0083F, 0x0083F}, + {0x0085C, 0x0085D}, + {0x0085F, 0x0085F}, + {0x0086B, 0x0086F}, + {0x0088F, 0x0088F}, + {0x00892, 0x00896}, + {0x00984, 0x00984}, + {0x0098D, 0x0098E}, + {0x00991, 0x00992}, + {0x009A9, 0x009A9}, + {0x009B1, 0x009B1}, + {0x009B3, 0x009B5}, + {0x009BA, 0x009BB}, + {0x009C5, 0x009C6}, + {0x009C9, 0x009CA}, + {0x009CF, 0x009D6}, + {0x009D8, 0x009DB}, + {0x009DE, 0x009DE}, + {0x009E4, 0x009E5}, + {0x009FF, 0x00A00}, + {0x00A04, 0x00A04}, + {0x00A0B, 0x00A0E}, + {0x00A11, 0x00A12}, + {0x00A29, 0x00A29}, + {0x00A31, 0x00A31}, + {0x00A34, 0x00A34}, + {0x00A37, 0x00A37}, + {0x00A3A, 0x00A3B}, + {0x00A3D, 0x00A3D}, + {0x00A43, 0x00A46}, + {0x00A49, 0x00A4A}, + {0x00A4E, 0x00A50}, + {0x00A52, 0x00A58}, + {0x00A5D, 0x00A5D}, + {0x00A5F, 0x00A65}, + {0x00A77, 0x00A80}, + {0x00A84, 0x00A84}, + {0x00A8E, 0x00A8E}, + {0x00A92, 0x00A92}, + {0x00AA9, 0x00AA9}, + {0x00AB1, 0x00AB1}, + {0x00AB4, 0x00AB4}, + {0x00ABA, 0x00ABB}, + {0x00AC6, 0x00AC6}, + {0x00ACA, 0x00ACA}, + {0x00ACE, 0x00ACF}, + {0x00AD1, 0x00ADF}, + {0x00AE4, 0x00AE5}, + {0x00AF2, 0x00AF8}, + {0x00B00, 0x00B00}, + {0x00B04, 0x00B04}, + {0x00B0D, 0x00B0E}, + {0x00B11, 0x00B12}, + {0x00B29, 0x00B29}, + {0x00B31, 0x00B31}, + {0x00B34, 0x00B34}, + {0x00B3A, 0x00B3B}, + {0x00B45, 0x00B46}, + {0x00B49, 0x00B4A}, + {0x00B4E, 0x00B54}, + {0x00B58, 0x00B5B}, + {0x00B5E, 0x00B5E}, + {0x00B64, 0x00B65}, + {0x00B78, 0x00B81}, + {0x00B84, 0x00B84}, + {0x00B8B, 0x00B8D}, + {0x00B91, 0x00B91}, + {0x00B96, 0x00B98}, + {0x00B9B, 0x00B9B}, + {0x00B9D, 0x00B9D}, + {0x00BA0, 0x00BA2}, + {0x00BA5, 0x00BA7}, + {0x00BAB, 0x00BAD}, + {0x00BBA, 0x00BBD}, + {0x00BC3, 0x00BC5}, + {0x00BC9, 0x00BC9}, + {0x00BCE, 0x00BCF}, + {0x00BD1, 0x00BD6}, + {0x00BD8, 0x00BE5}, + {0x00BFB, 0x00BFF}, + {0x00C0D, 0x00C0D}, + {0x00C11, 0x00C11}, + {0x00C29, 0x00C29}, + {0x00C3A, 0x00C3B}, + {0x00C45, 0x00C45}, + {0x00C49, 0x00C49}, + {0x00C4E, 0x00C54}, + {0x00C57, 0x00C57}, + {0x00C5B, 0x00C5C}, + {0x00C5E, 0x00C5F}, + {0x00C64, 0x00C65}, + {0x00C70, 0x00C76}, + {0x00C8D, 0x00C8D}, + {0x00C91, 0x00C91}, + {0x00CA9, 0x00CA9}, + {0x00CB4, 0x00CB4}, + {0x00CBA, 0x00CBB}, + {0x00CC5, 0x00CC5}, + {0x00CC9, 0x00CC9}, + {0x00CCE, 0x00CD4}, + {0x00CD7, 0x00CDC}, + {0x00CDF, 0x00CDF}, + {0x00CE4, 0x00CE5}, + {0x00CF0, 0x00CF0}, + {0x00CF4, 0x00CFF}, + {0x00D0D, 0x00D0D}, + {0x00D11, 0x00D11}, + {0x00D45, 0x00D45}, + {0x00D49, 0x00D49}, + {0x00D50, 0x00D53}, + {0x00D64, 0x00D65}, + {0x00D80, 0x00D80}, + {0x00D84, 0x00D84}, + {0x00D97, 0x00D99}, + {0x00DB2, 0x00DB2}, + {0x00DBC, 0x00DBC}, + {0x00DBE, 0x00DBF}, + {0x00DC7, 0x00DC9}, + {0x00DCB, 0x00DCE}, + {0x00DD5, 0x00DD5}, + {0x00DD7, 0x00DD7}, + {0x00DE0, 0x00DE5}, + {0x00DF0, 0x00DF1}, + {0x00DF5, 0x00E00}, + {0x00E3B, 0x00E3E}, + {0x00E5C, 0x00E80}, + {0x00E83, 0x00E83}, + {0x00E85, 0x00E85}, + {0x00E8B, 0x00E8B}, + {0x00EA4, 0x00EA4}, + {0x00EA6, 0x00EA6}, + {0x00EBE, 0x00EBF}, + {0x00EC5, 0x00EC5}, + {0x00EC7, 0x00EC7}, + {0x00ECF, 0x00ECF}, + {0x00EDA, 0x00EDB}, + {0x00EE0, 0x00EFF}, + {0x00F48, 0x00F48}, + {0x00F6D, 0x00F70}, + {0x00F98, 0x00F98}, + {0x00FBD, 0x00FBD}, + {0x00FCD, 0x00FCD}, + {0x00FDB, 0x00FFF}, + {0x010C6, 0x010C6}, + {0x010C8, 0x010CC}, + {0x010CE, 0x010CF}, + {0x01249, 0x01249}, + {0x0124E, 0x0124F}, + {0x01257, 0x01257}, + {0x01259, 0x01259}, + {0x0125E, 0x0125F}, + {0x01289, 0x01289}, + {0x0128E, 0x0128F}, + {0x012B1, 0x012B1}, + {0x012B6, 0x012B7}, + {0x012BF, 0x012BF}, + {0x012C1, 0x012C1}, + {0x012C6, 0x012C7}, + {0x012D7, 0x012D7}, + {0x01311, 0x01311}, + {0x01316, 0x01317}, + {0x0135B, 0x0135C}, + {0x0137D, 0x0137F}, + {0x0139A, 0x0139F}, + {0x013F6, 0x013F7}, + {0x013FE, 0x013FF}, + {0x0169D, 0x0169F}, + {0x016F9, 0x016FF}, + {0x01716, 0x0171E}, + {0x01737, 0x0173F}, + {0x01754, 0x0175F}, + {0x0176D, 0x0176D}, + {0x01771, 0x01771}, + {0x01774, 0x0177F}, + {0x017DE, 0x017DF}, + {0x017EA, 0x017EF}, + {0x017FA, 0x017FF}, + {0x0181A, 0x0181F}, + {0x01879, 0x0187F}, + {0x018AB, 0x018AF}, + {0x018F6, 0x018FF}, + {0x0191F, 0x0191F}, + {0x0192C, 0x0192F}, + {0x0193C, 0x0193F}, + {0x01941, 0x01943}, + {0x0196E, 0x0196F}, + {0x01975, 0x0197F}, + {0x019AC, 0x019AF}, + {0x019CA, 0x019CF}, + {0x019DB, 0x019DD}, + {0x01A1C, 0x01A1D}, + {0x01A5F, 0x01A5F}, + {0x01A7D, 0x01A7E}, + {0x01A8A, 0x01A8F}, + {0x01A9A, 0x01A9F}, + {0x01AAE, 0x01AAF}, + {0x01ACF, 0x01AFF}, + {0x01B4D, 0x01B4D}, + {0x01BF4, 0x01BFB}, + {0x01C38, 0x01C3A}, + {0x01C4A, 0x01C4C}, + {0x01C8B, 0x01C8F}, + {0x01CBB, 0x01CBC}, + {0x01CC8, 0x01CCF}, + {0x01CFB, 0x01CFF}, + {0x01F16, 0x01F17}, + {0x01F1E, 0x01F1F}, + {0x01F46, 0x01F47}, + {0x01F4E, 0x01F4F}, + {0x01F58, 0x01F58}, + {0x01F5A, 0x01F5A}, + {0x01F5C, 0x01F5C}, + {0x01F5E, 0x01F5E}, + {0x01F7E, 0x01F7F}, + {0x01FB5, 0x01FB5}, + {0x01FC5, 0x01FC5}, + {0x01FD4, 0x01FD5}, + {0x01FDC, 0x01FDC}, + {0x01FF0, 0x01FF1}, + {0x01FF5, 0x01FF5}, + {0x01FFF, 0x01FFF}, + {0x02065, 0x02065}, + {0x02072, 0x02073}, + {0x0208F, 0x0208F}, + {0x0209D, 0x0209F}, + {0x020C1, 0x020CF}, + {0x020F1, 0x020FF}, + {0x0218C, 0x0218F}, + {0x0242A, 0x0243F}, + {0x0244B, 0x0245F}, + {0x02B74, 0x02B75}, + {0x02B96, 0x02B96}, + {0x02CF4, 0x02CF8}, + {0x02D26, 0x02D26}, + {0x02D28, 0x02D2C}, + {0x02D2E, 0x02D2F}, + {0x02D68, 0x02D6E}, + {0x02D71, 0x02D7E}, + {0x02D97, 0x02D9F}, + {0x02DA7, 0x02DA7}, + {0x02DAF, 0x02DAF}, + {0x02DB7, 0x02DB7}, + {0x02DBF, 0x02DBF}, + {0x02DC7, 0x02DC7}, + {0x02DCF, 0x02DCF}, + {0x02DD7, 0x02DD7}, + {0x02DDF, 0x02DDF}, + {0x02E5E, 0x02E7F}, + {0x02E9A, 0x02E9A}, + {0x02EF4, 0x02EFF}, + {0x02FD6, 0x02FEF}, + {0x03040, 0x03040}, + {0x03097, 0x03098}, + {0x03100, 0x03104}, + {0x03130, 0x03130}, + {0x0318F, 0x0318F}, + {0x031E6, 0x031EE}, + {0x0321F, 0x0321F}, + {0x03401, 0x04DBE}, + {0x04E01, 0x09FFE}, + {0x0A48D, 0x0A48F}, + {0x0A4C7, 0x0A4CF}, + {0x0A62C, 0x0A63F}, + {0x0A6F8, 0x0A6FF}, + {0x0A7CE, 0x0A7CF}, + {0x0A7D2, 0x0A7D2}, + {0x0A7D4, 0x0A7D4}, + {0x0A7DD, 0x0A7F1}, + {0x0A82D, 0x0A82F}, + {0x0A83A, 0x0A83F}, + {0x0A878, 0x0A87F}, + {0x0A8C6, 0x0A8CD}, + {0x0A8DA, 0x0A8DF}, + {0x0A954, 0x0A95E}, + {0x0A97D, 0x0A97F}, + {0x0A9CE, 0x0A9CE}, + {0x0A9DA, 0x0A9DD}, + {0x0A9FF, 0x0A9FF}, + {0x0AA37, 0x0AA3F}, + {0x0AA4E, 0x0AA4F}, + {0x0AA5A, 0x0AA5B}, + {0x0AAC3, 0x0AADA}, + {0x0AAF7, 0x0AB00}, + {0x0AB07, 0x0AB08}, + {0x0AB0F, 0x0AB10}, + {0x0AB17, 0x0AB1F}, + {0x0AB27, 0x0AB27}, + {0x0AB2F, 0x0AB2F}, + {0x0AB6C, 0x0AB6F}, + {0x0ABEE, 0x0ABEF}, + {0x0ABFA, 0x0ABFF}, + {0x0AC01, 0x0D7A2}, + {0x0D7A4, 0x0D7AF}, + {0x0D7C7, 0x0D7CA}, + {0x0D7FC, 0x0D7FF}, + {0x0FA6E, 0x0FA6F}, + {0x0FADA, 0x0FAFF}, + {0x0FB07, 0x0FB12}, + {0x0FB18, 0x0FB1C}, + {0x0FB37, 0x0FB37}, + {0x0FB3D, 0x0FB3D}, + {0x0FB3F, 0x0FB3F}, + {0x0FB42, 0x0FB42}, + {0x0FB45, 0x0FB45}, + {0x0FBC3, 0x0FBD2}, + {0x0FD90, 0x0FD91}, + {0x0FDC8, 0x0FDCE}, + {0x0FE1A, 0x0FE1F}, + {0x0FE53, 0x0FE53}, + {0x0FE67, 0x0FE67}, + {0x0FE6C, 0x0FE6F}, + {0x0FE75, 0x0FE75}, + {0x0FEFD, 0x0FEFE}, + {0x0FF00, 0x0FF00}, + {0x0FFBF, 0x0FFC1}, + {0x0FFC8, 0x0FFC9}, + {0x0FFD0, 0x0FFD1}, + {0x0FFD8, 0x0FFD9}, + {0x0FFDD, 0x0FFDF}, + {0x0FFE7, 0x0FFE7}, + {0x0FFEF, 0x0FFF8}, + {0x1000C, 0x1000C}, + {0x10027, 0x10027}, + {0x1003B, 0x1003B}, + {0x1003E, 0x1003E}, + {0x1004E, 0x1004F}, + {0x1005E, 0x1007F}, + {0x100FB, 0x100FF}, + {0x10103, 0x10106}, + {0x10134, 0x10136}, + {0x1018F, 0x1018F}, + {0x1019D, 0x1019F}, + {0x101A1, 0x101CF}, + {0x101FE, 0x1027F}, + {0x1029D, 0x1029F}, + {0x102D1, 0x102DF}, + {0x102FC, 0x102FF}, + {0x10324, 0x1032C}, + {0x1034B, 0x1034F}, + {0x1037B, 0x1037F}, + {0x1039E, 0x1039E}, + {0x103C4, 0x103C7}, + {0x103D6, 0x103FF}, + {0x1049E, 0x1049F}, + {0x104AA, 0x104AF}, + {0x104D4, 0x104D7}, + {0x104FC, 0x104FF}, + {0x10528, 0x1052F}, + {0x10564, 0x1056E}, + {0x1057B, 0x1057B}, + {0x1058B, 0x1058B}, + {0x10593, 0x10593}, + {0x10596, 0x10596}, + {0x105A2, 0x105A2}, + {0x105B2, 0x105B2}, + {0x105BA, 0x105BA}, + {0x105BD, 0x105BF}, + {0x105F4, 0x105FF}, + {0x10737, 0x1073F}, + {0x10756, 0x1075F}, + {0x10768, 0x1077F}, + {0x10786, 0x10786}, + {0x107B1, 0x107B1}, + {0x107BB, 0x107FF}, + {0x10806, 0x10807}, + {0x10809, 0x10809}, + {0x10836, 0x10836}, + {0x10839, 0x1083B}, + {0x1083D, 0x1083E}, + {0x10856, 0x10856}, + {0x1089F, 0x108A6}, + {0x108B0, 0x108DF}, + {0x108F3, 0x108F3}, + {0x108F6, 0x108FA}, + {0x1091C, 0x1091E}, + {0x1093A, 0x1093E}, + {0x10940, 0x1097F}, + {0x109B8, 0x109BB}, + {0x109D0, 0x109D1}, + {0x10A04, 0x10A04}, + {0x10A07, 0x10A0B}, + {0x10A14, 0x10A14}, + {0x10A18, 0x10A18}, + {0x10A36, 0x10A37}, + {0x10A3B, 0x10A3E}, + {0x10A49, 0x10A4F}, + {0x10A59, 0x10A5F}, + {0x10AA0, 0x10ABF}, + {0x10AE7, 0x10AEA}, + {0x10AF7, 0x10AFF}, + {0x10B36, 0x10B38}, + {0x10B56, 0x10B57}, + {0x10B73, 0x10B77}, + {0x10B92, 0x10B98}, + {0x10B9D, 0x10BA8}, + {0x10BB0, 0x10BFF}, + {0x10C49, 0x10C7F}, + {0x10CB3, 0x10CBF}, + {0x10CF3, 0x10CF9}, + {0x10D28, 0x10D2F}, + {0x10D3A, 0x10D3F}, + {0x10D66, 0x10D68}, + {0x10D86, 0x10D8D}, + {0x10D90, 0x10E5F}, + {0x10E7F, 0x10E7F}, + {0x10EAA, 0x10EAA}, + {0x10EAE, 0x10EAF}, + {0x10EB2, 0x10EC1}, + {0x10EC5, 0x10EFB}, + {0x10F28, 0x10F2F}, + {0x10F5A, 0x10F6F}, + {0x10F8A, 0x10FAF}, + {0x10FCC, 0x10FDF}, + {0x10FF7, 0x10FFF}, + {0x1104E, 0x11051}, + {0x11076, 0x1107E}, + {0x110C3, 0x110CC}, + {0x110CE, 0x110CF}, + {0x110E9, 0x110EF}, + {0x110FA, 0x110FF}, + {0x11135, 0x11135}, + {0x11148, 0x1114F}, + {0x11177, 0x1117F}, + {0x111E0, 0x111E0}, + {0x111F5, 0x111FF}, + {0x11212, 0x11212}, + {0x11242, 0x1127F}, + {0x11287, 0x11287}, + {0x11289, 0x11289}, + {0x1128E, 0x1128E}, + {0x1129E, 0x1129E}, + {0x112AA, 0x112AF}, + {0x112EB, 0x112EF}, + {0x112FA, 0x112FF}, + {0x11304, 0x11304}, + {0x1130D, 0x1130E}, + {0x11311, 0x11312}, + {0x11329, 0x11329}, + {0x11331, 0x11331}, + {0x11334, 0x11334}, + {0x1133A, 0x1133A}, + {0x11345, 0x11346}, + {0x11349, 0x1134A}, + {0x1134E, 0x1134F}, + {0x11351, 0x11356}, + {0x11358, 0x1135C}, + {0x11364, 0x11365}, + {0x1136D, 0x1136F}, + {0x11375, 0x1137F}, + {0x1138A, 0x1138A}, + {0x1138C, 0x1138D}, + {0x1138F, 0x1138F}, + {0x113B6, 0x113B6}, + {0x113C1, 0x113C1}, + {0x113C3, 0x113C4}, + {0x113C6, 0x113C6}, + {0x113CB, 0x113CB}, + {0x113D6, 0x113D6}, + {0x113D9, 0x113E0}, + {0x113E3, 0x113FF}, + {0x1145C, 0x1145C}, + {0x11462, 0x1147F}, + {0x114C8, 0x114CF}, + {0x114DA, 0x1157F}, + {0x115B6, 0x115B7}, + {0x115DE, 0x115FF}, + {0x11645, 0x1164F}, + {0x1165A, 0x1165F}, + {0x1166D, 0x1167F}, + {0x116BA, 0x116BF}, + {0x116CA, 0x116CF}, + {0x116E4, 0x116FF}, + {0x1171B, 0x1171C}, + {0x1172C, 0x1172F}, + {0x11747, 0x117FF}, + {0x1183C, 0x1189F}, + {0x118F3, 0x118FE}, + {0x11907, 0x11908}, + {0x1190A, 0x1190B}, + {0x11914, 0x11914}, + {0x11917, 0x11917}, + {0x11936, 0x11936}, + {0x11939, 0x1193A}, + {0x11947, 0x1194F}, + {0x1195A, 0x1199F}, + {0x119A8, 0x119A9}, + {0x119D8, 0x119D9}, + {0x119E5, 0x119FF}, + {0x11A48, 0x11A4F}, + {0x11AA3, 0x11AAF}, + {0x11AF9, 0x11AFF}, + {0x11B0A, 0x11BBF}, + {0x11BE2, 0x11BEF}, + {0x11BFA, 0x11BFF}, + {0x11C09, 0x11C09}, + {0x11C37, 0x11C37}, + {0x11C46, 0x11C4F}, + {0x11C6D, 0x11C6F}, + {0x11C90, 0x11C91}, + {0x11CA8, 0x11CA8}, + {0x11CB7, 0x11CFF}, + {0x11D07, 0x11D07}, + {0x11D0A, 0x11D0A}, + {0x11D37, 0x11D39}, + {0x11D3B, 0x11D3B}, + {0x11D3E, 0x11D3E}, + {0x11D48, 0x11D4F}, + {0x11D5A, 0x11D5F}, + {0x11D66, 0x11D66}, + {0x11D69, 0x11D69}, + {0x11D8F, 0x11D8F}, + {0x11D92, 0x11D92}, + {0x11D99, 0x11D9F}, + {0x11DAA, 0x11EDF}, + {0x11EF9, 0x11EFF}, + {0x11F11, 0x11F11}, + {0x11F3B, 0x11F3D}, + {0x11F5B, 0x11FAF}, + {0x11FB1, 0x11FBF}, + {0x11FF2, 0x11FFE}, + {0x1239A, 0x123FF}, + {0x1246F, 0x1246F}, + {0x12475, 0x1247F}, + {0x12544, 0x12F8F}, + {0x12FF3, 0x12FFF}, + {0x13456, 0x1345F}, + {0x143FB, 0x143FF}, + {0x14647, 0x160FF}, + {0x1613A, 0x167FF}, + {0x16A39, 0x16A3F}, + {0x16A5F, 0x16A5F}, + {0x16A6A, 0x16A6D}, + {0x16ABF, 0x16ABF}, + {0x16ACA, 0x16ACF}, + {0x16AEE, 0x16AEF}, + {0x16AF6, 0x16AFF}, + {0x16B46, 0x16B4F}, + {0x16B5A, 0x16B5A}, + {0x16B62, 0x16B62}, + {0x16B78, 0x16B7C}, + {0x16B90, 0x16D3F}, + {0x16D7A, 0x16E3F}, + {0x16E9B, 0x16EFF}, + {0x16F4B, 0x16F4E}, + {0x16F88, 0x16F8E}, + {0x16FA0, 0x16FDF}, + {0x16FE5, 0x16FEF}, + {0x16FF2, 0x16FFF}, + {0x17001, 0x187F6}, + {0x187F8, 0x187FF}, + {0x18CD6, 0x18CFE}, + {0x18D01, 0x18D07}, + {0x18D09, 0x1AFEF}, + {0x1AFF4, 0x1AFF4}, + {0x1AFFC, 0x1AFFC}, + {0x1AFFF, 0x1AFFF}, + {0x1B123, 0x1B131}, + {0x1B133, 0x1B14F}, + {0x1B153, 0x1B154}, + {0x1B156, 0x1B163}, + {0x1B168, 0x1B16F}, + {0x1B2FC, 0x1BBFF}, + {0x1BC6B, 0x1BC6F}, + {0x1BC7D, 0x1BC7F}, + {0x1BC89, 0x1BC8F}, + {0x1BC9A, 0x1BC9B}, + {0x1BCA4, 0x1CBFF}, + {0x1CCFA, 0x1CCFF}, + {0x1CEB4, 0x1CEFF}, + {0x1CF2E, 0x1CF2F}, + {0x1CF47, 0x1CF4F}, + {0x1CFC4, 0x1CFFF}, + {0x1D0F6, 0x1D0FF}, + {0x1D127, 0x1D128}, + {0x1D1EB, 0x1D1FF}, + {0x1D246, 0x1D2BF}, + {0x1D2D4, 0x1D2DF}, + {0x1D2F4, 0x1D2FF}, + {0x1D357, 0x1D35F}, + {0x1D379, 0x1D3FF}, + {0x1D455, 0x1D455}, + {0x1D49D, 0x1D49D}, + {0x1D4A0, 0x1D4A1}, + {0x1D4A3, 0x1D4A4}, + {0x1D4A7, 0x1D4A8}, + {0x1D4AD, 0x1D4AD}, + {0x1D4BA, 0x1D4BA}, + {0x1D4BC, 0x1D4BC}, + {0x1D4C4, 0x1D4C4}, + {0x1D506, 0x1D506}, + {0x1D50B, 0x1D50C}, + {0x1D515, 0x1D515}, + {0x1D51D, 0x1D51D}, + {0x1D53A, 0x1D53A}, + {0x1D53F, 0x1D53F}, + {0x1D545, 0x1D545}, + {0x1D547, 0x1D549}, + {0x1D551, 0x1D551}, + {0x1D6A6, 0x1D6A7}, + {0x1D7CC, 0x1D7CD}, + {0x1DA8C, 0x1DA9A}, + {0x1DAA0, 0x1DAA0}, + {0x1DAB0, 0x1DEFF}, + {0x1DF1F, 0x1DF24}, + {0x1DF2B, 0x1DFFF}, + {0x1E007, 0x1E007}, + {0x1E019, 0x1E01A}, + {0x1E022, 0x1E022}, + {0x1E025, 0x1E025}, + {0x1E02B, 0x1E02F}, + {0x1E06E, 0x1E08E}, + {0x1E090, 0x1E0FF}, + {0x1E12D, 0x1E12F}, + {0x1E13E, 0x1E13F}, + {0x1E14A, 0x1E14D}, + {0x1E150, 0x1E28F}, + {0x1E2AF, 0x1E2BF}, + {0x1E2FA, 0x1E2FE}, + {0x1E300, 0x1E4CF}, + {0x1E4FA, 0x1E5CF}, + {0x1E5FB, 0x1E5FE}, + {0x1E600, 0x1E7DF}, + {0x1E7E7, 0x1E7E7}, + {0x1E7EC, 0x1E7EC}, + {0x1E7EF, 0x1E7EF}, + {0x1E7FF, 0x1E7FF}, + {0x1E8C5, 0x1E8C6}, + {0x1E8D7, 0x1E8FF}, + {0x1E94C, 0x1E94F}, + {0x1E95A, 0x1E95D}, + {0x1E960, 0x1EC70}, + {0x1ECB5, 0x1ED00}, + {0x1ED3E, 0x1EDFF}, + {0x1EE04, 0x1EE04}, + {0x1EE20, 0x1EE20}, + {0x1EE23, 0x1EE23}, + {0x1EE25, 0x1EE26}, + {0x1EE28, 0x1EE28}, + {0x1EE33, 0x1EE33}, + {0x1EE38, 0x1EE38}, + {0x1EE3A, 0x1EE3A}, + {0x1EE3C, 0x1EE41}, + {0x1EE43, 0x1EE46}, + {0x1EE48, 0x1EE48}, + {0x1EE4A, 0x1EE4A}, + {0x1EE4C, 0x1EE4C}, + {0x1EE50, 0x1EE50}, + {0x1EE53, 0x1EE53}, + {0x1EE55, 0x1EE56}, + {0x1EE58, 0x1EE58}, + {0x1EE5A, 0x1EE5A}, + {0x1EE5C, 0x1EE5C}, + {0x1EE5E, 0x1EE5E}, + {0x1EE60, 0x1EE60}, + {0x1EE63, 0x1EE63}, + {0x1EE65, 0x1EE66}, + {0x1EE6B, 0x1EE6B}, + {0x1EE73, 0x1EE73}, + {0x1EE78, 0x1EE78}, + {0x1EE7D, 0x1EE7D}, + {0x1EE7F, 0x1EE7F}, + {0x1EE8A, 0x1EE8A}, + {0x1EE9C, 0x1EEA0}, + {0x1EEA4, 0x1EEA4}, + {0x1EEAA, 0x1EEAA}, + {0x1EEBC, 0x1EEEF}, + {0x1EEF2, 0x1EFFF}, + {0x1F02C, 0x1F02F}, + {0x1F094, 0x1F09F}, + {0x1F0AF, 0x1F0B0}, + {0x1F0C0, 0x1F0C0}, + {0x1F0D0, 0x1F0D0}, + {0x1F0F6, 0x1F0FF}, + {0x1F1AE, 0x1F1E5}, + {0x1F203, 0x1F20F}, + {0x1F23C, 0x1F23F}, + {0x1F249, 0x1F24F}, + {0x1F252, 0x1F25F}, + {0x1F266, 0x1F2FF}, + {0x1F6D8, 0x1F6DB}, + {0x1F6ED, 0x1F6EF}, + {0x1F6FD, 0x1F6FF}, + {0x1F777, 0x1F77A}, + {0x1F7DA, 0x1F7DF}, + {0x1F7EC, 0x1F7EF}, + {0x1F7F1, 0x1F7FF}, + {0x1F80C, 0x1F80F}, + {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, + {0x1F888, 0x1F88F}, + {0x1F8AE, 0x1F8AF}, + {0x1F8BC, 0x1F8BF}, + {0x1F8C2, 0x1F8FF}, + {0x1FA54, 0x1FA5F}, + {0x1FA6E, 0x1FA6F}, + {0x1FA7D, 0x1FA7F}, + {0x1FA8A, 0x1FA8E}, + {0x1FAC7, 0x1FACD}, + {0x1FADD, 0x1FADE}, + {0x1FAEA, 0x1FAEF}, + {0x1FAF9, 0x1FAFF}, + {0x1FB93, 0x1FB93}, + {0x1FBFA, 0x1FFFD}, + {0x20001, 0x2A6DE}, + {0x2A6E0, 0x2A6FF}, + {0x2A701, 0x2B738}, + {0x2B73A, 0x2B73F}, + {0x2B741, 0x2B81C}, + {0x2B81E, 0x2B81F}, + {0x2B821, 0x2CEA0}, + {0x2CEA2, 0x2CEAF}, + {0x2CEB1, 0x2EBDF}, + {0x2EBE1, 0x2EBEF}, + {0x2EBF1, 0x2EE5C}, + {0x2EE5E, 0x2F7FF}, + {0x2FA1E, 0x2FFFD}, + {0x30001, 0x31349}, + {0x3134B, 0x3134F}, + {0x31351, 0x323AE}, + {0x323B0, 0x3FFFD}, + {0x40000, 0x4FFFD}, + {0x50000, 0x5FFFD}, + {0x60000, 0x6FFFD}, + {0x70000, 0x7FFFD}, + {0x80000, 0x8FFFD}, + {0x90000, 0x9FFFD}, + {0xA0000, 0xAFFFD}, + {0xB0000, 0xBFFFD}, + {0xC0000, 0xCFFFD}, + {0xD0000, 0xDFFFD}, + {0xE0000, 0xE0000}, + {0xE0002, 0xE001F}, + {0xE0080, 0xE00FF}, + {0xE01F0, 0xEFFFD} +}; + +/* Non-characters. */ +static const struct widechar_range widechar_nonchar_table[] = { + {0x0FDD0, 0x0FDEF}, + {0x0FFFE, 0x0FFFF}, + {0x1FFFE, 0x1FFFF}, + {0x2FFFE, 0x2FFFF}, + {0x3FFFE, 0x3FFFF}, + {0x4FFFE, 0x4FFFF}, + {0x5FFFE, 0x5FFFF}, + {0x6FFFE, 0x6FFFF}, + {0x7FFFE, 0x7FFFF}, + {0x8FFFE, 0x8FFFF}, + {0x9FFFE, 0x9FFFF}, + {0xAFFFE, 0xAFFFF}, + {0xBFFFE, 0xBFFFF}, + {0xCFFFE, 0xCFFFF}, + {0xDFFFE, 0xDFFFF}, + {0xEFFFE, 0xEFFFF}, + {0xFFFFE, 0xFFFFF}, + {0x10FFFE, 0x10FFFF} +}; + +/* Characters that were widened from width 1 to 2 in Unicode 9. */ +static const struct widechar_range widechar_widened_table[] = { + {0x0231A, 0x0231B}, + {0x023E9, 0x023EC}, + {0x023F0, 0x023F0}, + {0x023F3, 0x023F3}, + {0x025FD, 0x025FE}, + {0x02614, 0x02615}, + {0x02648, 0x02653}, + {0x0267F, 0x0267F}, + {0x02693, 0x02693}, + {0x026A1, 0x026A1}, + {0x026AA, 0x026AB}, + {0x026BD, 0x026BE}, + {0x026C4, 0x026C5}, + {0x026CE, 0x026CE}, + {0x026D4, 0x026D4}, + {0x026EA, 0x026EA}, + {0x026F2, 0x026F3}, + {0x026F5, 0x026F5}, + {0x026FA, 0x026FA}, + {0x026FD, 0x026FD}, + {0x02705, 0x02705}, + {0x0270A, 0x0270B}, + {0x02728, 0x02728}, + {0x0274C, 0x0274C}, + {0x0274E, 0x0274E}, + {0x02753, 0x02755}, + {0x02757, 0x02757}, + {0x02795, 0x02797}, + {0x027B0, 0x027B0}, + {0x027BF, 0x027BF}, + {0x02B1B, 0x02B1C}, + {0x02B50, 0x02B50}, + {0x02B55, 0x02B55}, + {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, + {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, + {0x1F201, 0x1F201}, + {0x1F21A, 0x1F21A}, + {0x1F22F, 0x1F22F}, + {0x1F232, 0x1F236}, + {0x1F238, 0x1F23A}, + {0x1F250, 0x1F251}, + {0x1F300, 0x1F320}, + {0x1F32D, 0x1F335}, + {0x1F337, 0x1F37C}, + {0x1F37E, 0x1F393}, + {0x1F3A0, 0x1F3CA}, + {0x1F3CF, 0x1F3D3}, + {0x1F3E0, 0x1F3F0}, + {0x1F3F4, 0x1F3F4}, + {0x1F3F8, 0x1F43E}, + {0x1F440, 0x1F440}, + {0x1F442, 0x1F4FC}, + {0x1F4FF, 0x1F53D}, + {0x1F54B, 0x1F54E}, + {0x1F550, 0x1F567}, + {0x1F595, 0x1F596}, + {0x1F5FB, 0x1F64F}, + {0x1F680, 0x1F6C5}, + {0x1F6CC, 0x1F6CC}, + {0x1F6D0, 0x1F6D0}, + {0x1F6EB, 0x1F6EC}, + {0x1F910, 0x1F918}, + {0x1F980, 0x1F984}, + {0x1F9C0, 0x1F9C0} +}; + +template +bool widechar_in_table(const Collection &arr, uint32_t c) { + auto where = std::lower_bound(std::begin(arr), std::end(arr), c, + [](widechar_range p, uint32_t c) { return p.hi < c; }); + return where != std::end(arr) && where->lo <= c; +} + +/* Return the width of character c, or a special negative value. */ +int widechar_wcwidth(uint32_t c) { + if (widechar_in_table(widechar_ascii_table, c)) + return 1; + if (widechar_in_table(widechar_private_table, c)) + return widechar_private_use; + if (widechar_in_table(widechar_nonprint_table, c)) + return widechar_nonprint; + if (widechar_in_table(widechar_nonchar_table, c)) + return widechar_non_character; + if (widechar_in_table(widechar_combining_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_combiningletters_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_doublewide_table, c)) + return 2; + if (widechar_in_table(widechar_ambiguous_table, c)) + return widechar_ambiguous; + if (widechar_in_table(widechar_unassigned_table, c)) + return widechar_unassigned; + if (widechar_in_table(widechar_widened_table, c)) + return widechar_widened_in_9; + return 1; +} + +} // namespace +#endif // WIDECHAR_WIDTH_H From 1e2cace5f1dbd55ed76afbad7519100b76af5d88 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 28 Dec 2024 23:40:25 +0100 Subject: [PATCH 172/243] fix documentation of substring --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f19dd473a..a0e2753b5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4059,7 +4059,7 @@ static RegisterPrimOp primop_toString({ }); /* `substring start len str' returns the substring of `str' starting - at character position `min(start, stringLength str)' inclusive and + at byte position `min(start, stringLength str)' inclusive and ending at `min(start + len, stringLength str)'. `start' must be non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -4098,7 +4098,7 @@ static RegisterPrimOp primop_substring({ .name = "__substring", .args = {"start", "len", "s"}, .doc = R"( - Return the substring of *s* from character position *start* + Return the substring of *s* from byte position *start* (zero-based) up to but not including *start + len*. If *start* is greater than the length of the string, an empty string is returned. If *start + len* lies beyond the end of the string or *len* is `-1`, From b5f10655ede1c5f8986f622de7ad73daeda7d5fc Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:56:53 +0100 Subject: [PATCH 173/243] ci: update Ubuntu runner to ubuntu-24.04 Link: https://github.com/actions/runner-images/issues/10636 --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/labels.yml | 2 +- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2426ab166..7592f60b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: read-all jobs: eval: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-latest] + os: [ubuntu-24.04, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -37,7 +37,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-latest] + os: [ubuntu-24.04, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -68,7 +68,7 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' - run: brew install fish if: matrix.os == 'macos-latest' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" @@ -86,7 +86,7 @@ jobs: permissions: contents: none name: Check Docker secrets present for installer tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: docker: ${{ steps.secret.outputs.docker }} steps: @@ -106,7 +106,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && github.ref_name == 'master' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check for secrets id: secret @@ -158,7 +158,7 @@ jobs: docker push $IMAGE_ID:master vm_tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main @@ -173,7 +173,7 @@ jobs: flake_regressions: needs: vm_tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout nix uses: actions/checkout@v4 diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 9d2ac80a3..23a5d9e51 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -15,7 +15,7 @@ permissions: jobs: labels: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.repository_owner == 'NixOS' steps: - uses: actions/labeler@v5 diff --git a/.mergify.yml b/.mergify.yml index c2e115487..827e97290 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,9 @@ queue_rules: # all required tests need to go here merge_conditions: - check-success=tests (macos-latest) - - check-success=tests (ubuntu-22.04) + - check-success=tests (ubuntu-24.04) - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-22.04) + - check-success=installer_test (ubuntu-24.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 7e8762fe0..082863bcc 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-22.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From ce1e9ba85a1937ba695aab4da34bf310c5a8ecc9 Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 02:14:55 +0100 Subject: [PATCH 174/243] ci: lock macOS runner to macos-14 --- .github/workflows/ci.yml | 6 +++--- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7592f60b9..cd567cd63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest] + os: [ubuntu-24.04, macos-14] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest] + os: [ubuntu-24.04, macos-14] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -70,7 +70,7 @@ jobs: - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-24.04' - run: brew install fish - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-14' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" diff --git a/.mergify.yml b/.mergify.yml index 827e97290..13d2002ea 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,9 +2,9 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=tests (macos-latest) + - check-success=tests (macos-14) - check-success=tests (ubuntu-24.04) - - check-success=installer_test (macos-latest) + - check-success=installer_test (macos-14) - check-success=installer_test (ubuntu-24.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 082863bcc..d582ce4b4 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From 5c968be16288b7ad54b8c6d94cd0566be811ba50 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sun, 29 Dec 2024 16:05:16 +0000 Subject: [PATCH 175/243] m4/gcc_bug_80431.m4: drop unused file As autotools-based build system is gone the file is not used anymore. --- m4/gcc_bug_80431.m4 | 66 --------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 m4/gcc_bug_80431.m4 diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 deleted file mode 100644 index cdc4ddb40..000000000 --- a/m4/gcc_bug_80431.m4 +++ /dev/null @@ -1,66 +0,0 @@ -# Ensure that this bug is not present in the C++ toolchain we are using. -# -# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 -# -# The test program is from that issue, with only a slight modification -# to set an exit status instead of printing strings. -AC_DEFUN([ENSURE_NO_GCC_BUG_80431], -[ - AC_MSG_CHECKING([that GCC bug 80431 is fixed]) - AC_LANG_PUSH(C++) - AC_RUN_IFELSE( - [AC_LANG_PROGRAM( - [[ - #include - - static bool a = true; - static bool b = true; - - struct Options { }; - - struct Option - { - Option(Options * options) - { - a = false; - } - - ~Option() - { - b = false; - } - }; - - struct MyOptions : Options { }; - - struct MyOptions2 : virtual MyOptions - { - Option foo{this}; - }; - ]], - [[ - { - MyOptions2 opts; - } - return (a << 1) | b; - ]])], - [status_80431=0], - [status_80431=$?], - [status_80431='']) - AC_LANG_POP(C++) - AS_CASE([$status_80431], - [''],[ - AC_MSG_RESULT(cannot check because cross compiling) - AC_MSG_NOTICE(assume we are bug free) - ], - [0],[ - AC_MSG_RESULT(yes) - ], - [2],[ - AC_MSG_RESULT(no) - AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) - ], - [ - AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) - ]) -]) From edbfe863ce4ae4b89e554f29807e62674055f251 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sun, 29 Dec 2024 15:54:53 +0000 Subject: [PATCH 176/243] libcmd: update to support lowdown-1.4 API Upstream change https://github.com/kristapsdz/lowdown/commit/bab1d75079ac3866fc14e1fa4825ae64a94a7c0e moved a few fields from `lowdown_opts` toa new `lowdown_opts_term` struct. As a result the build started failing as: nix-cmd> [2/17] Compiling C++ object libnixcmd.so.p/markdown.cc.o nix-cmd> FAILED: libnixcmd.so.p/markdown.cc.o nix-cmd> g++ -Ilibnixcmd.so.p -I. -I.. -I/nix/store/b0bnrk5lacxbpgxgnc28r8q3wcazrgxj-nix-util-2.26.0pre-dev/include/nix -I/nix/store/cxnynq9ykyj4xxv6wf6dw7r0aw5x6n9k-libarchive-3.7.7-dev/include -I/nix/store/bfgjwkcb8snkizx578rzdahi75m8zyh4-nlohmann_json-3.11.3/include -I/nix/store/3sx8bq3sip6j2nv1m5xx4gbdp33v7iy6-nix-store-2.26.0pre-dev/include/nix -I/nix/store/sih2dgqzvsbv7p510lkfmas7s7wbsl4j-nix-fetchers-2.26.0pre-dev/include/nix -I/nix/store/68p8s20fsiiakj7nys7grbaixfnhsdzs-nix-expr-2.26.0pre-dev/include/nix -I/nix/store/gw7wknhzhfzzj9zww2kyi5xrzgf1ndki-boehm-gc-8.2.8-dev/include -I/nix/store/3jwb9j4vnsk5saq3wfyyp9il3mhs41l9-nix-flake-2.26.0pre-dev/include/nix -I/nix/store/8nwjvmq7m48v8g646jrxkikv6x47bc3m-nix-main-2.26.0pre-dev/include/nix -I/nix/store/rb0hzsw5wc1a7daizhpj824mbxlvijrq-lowdown-1.4.0-dev/include -I/nix/store/m388ywpk53fsp8r98brfd7nf1f5sskv0-editline-1.17.1-dev/include -fdiagnostics-color=always -D_GLIBCXX_ASSERTIONS=1 -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -std=c++2a -include config-util.hh -include config-store.hh -include config-expr.hh -include config-main.hh -include config-cmd.hh -Wdeprecated-copy -Werror=suggest-override -Werror=switch -Werror=switch-enum -Werror=unused-result -Wignored-qualifiers -Wimplicit-fallthrough -Wno-deprecated-declarations -O3 -fPIC -pthread -std=c++2a -std=c++2a -std=c++2a -std=c++2a -std=c++2a -std=c++2a -MD -MQ libnixcmd.so.p/markdown.cc.o -MF libnixcmd.so.p/markdown.cc.o.d -o libnixcmd.so.p/markdown.cc.o -c ../markdown.cc nix-cmd> ../markdown.cc: In function 'std::string nix::doRenderMarkdownToTerminal(std::string_view)': nix-cmd> ../markdown.cc:28:5: error: 'lowdown_opts' has no non-static data member named 'cols' nix-cmd> 28 | }; nix-cmd> | ^ The change adds version-based conditional to support both pre-1.4 and 1.4 forms of the initialization. Closes: https://github.com/NixOS/nix/issues/12113 --- src/libcmd/markdown.cc | 20 ++++++++++++++++---- src/libcmd/meson.build | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 6a0d05d9f..4566e6ba6 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown) { int windowWidth = getWindowSize().second; - struct lowdown_opts opts - { - .type = LOWDOWN_TERM, - .maxdepth = 20, +#if HAVE_LOWDOWN_1_4 + struct lowdown_opts_term opts_term { .cols = (size_t) std::max(windowWidth - 5, 60), .hmargin = 0, .vmargin = 0, + }; +#endif + struct lowdown_opts opts + { + .type = LOWDOWN_TERM, +#if HAVE_LOWDOWN_1_4 + .term = opts_term, +#endif + .maxdepth = 20, +#if !HAVE_LOWDOWN_1_4 + .cols = (size_t) std::max(windowWidth - 5, 60), + .hmargin = 0, + .vmargin = 0, +#endif .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, .oflags = LOWDOWN_TERM_NOLINK, }; diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 222817c81..914db0108 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -36,6 +36,8 @@ deps_public += nlohmann_json lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) deps_private += lowdown configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) +# The API changed slightly around terminal initialization. +configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int()) readline_flavor = get_option('readline-flavor') if readline_flavor == 'editline' From 1a402e0c53609df25539ae852ca7db6696e6f174 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 31 Dec 2024 13:52:18 +0100 Subject: [PATCH 177/243] test: test eval of newlines with raw output --- tests/functional/eval.nix | 2 +- tests/functional/eval.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/eval.nix b/tests/functional/eval.nix index befbd17a9..cabf28c29 100644 --- a/tests/functional/eval.nix +++ b/tests/functional/eval.nix @@ -1,5 +1,5 @@ { int = 123; - str = "foo"; + str = "foo\nbar"; attr.foo = "bar"; } diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 7af49d7fd..18f8589df 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -16,8 +16,8 @@ EOF nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval int -f "./eval.nix") == 123 ]] -[[ $(nix eval str -f "./eval.nix") == '"foo"' ]] -[[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] +[[ $(nix eval str -f "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix eval str --raw -f "./eval.nix") == $'foo\nbar' ]] [[ "$(nix eval attr -f "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix eval int -f - < "./eval.nix") == 123 ]] @@ -28,7 +28,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] -[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] +[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo\nbar"' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] From 7a8a28629c61c75af010ff0a5a88c16c4ce536c7 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 31 Dec 2024 14:04:50 +0100 Subject: [PATCH 178/243] feat(nix-instantiate): add --raw flag The experimental `nix eval` command already supports a `--raw` flag. This commit implements the same flag for the stable nix-instantiate command. Until now instructions and scripts that didn't want to rely on experimental features had to use workarounds such as: nix-instantiate --eval | tr -d \" (which also undesirably also removes double quotation marks within the string), or nix-instantiate --eval | jq -j (which undesirably depends on another package). Co-authored-by: Silvan Mosberger --- doc/manual/rl-next/nix-instantiate-raw.md | 8 ++++++++ doc/manual/source/command-ref/nix-copy-closure.md | 2 +- doc/manual/source/command-ref/nix-instantiate.md | 7 ++++++- src/nix-instantiate/nix-instantiate.cc | 10 ++++++++-- tests/functional/eval.sh | 1 + 5 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 doc/manual/rl-next/nix-instantiate-raw.md diff --git a/doc/manual/rl-next/nix-instantiate-raw.md b/doc/manual/rl-next/nix-instantiate-raw.md new file mode 100644 index 000000000..fb4a72b88 --- /dev/null +++ b/doc/manual/rl-next/nix-instantiate-raw.md @@ -0,0 +1,8 @@ +--- +synopsis: "`nix-instantiate --eval` now supports `--raw`" +prs: [12119] +--- + +The `nix-instantiate --eval` command now supports a `--raw` flag, when used +the evaluation result must be a string, which is printed verbatim without +quotation marks or escaping. diff --git a/doc/manual/source/command-ref/nix-copy-closure.md b/doc/manual/source/command-ref/nix-copy-closure.md index 8cfd6ebad..b7e31d93b 100644 --- a/doc/manual/source/command-ref/nix-copy-closure.md +++ b/doc/manual/source/command-ref/nix-copy-closure.md @@ -84,7 +84,7 @@ When using public key authentication, you can avoid typing the passphrase with ` > Copy GNU Hello from a remote machine using a known store path, and run it: > > ```shell-session -> $ storePath="$(nix-instantiate --eval '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')" +> $ storePath="$(nix-instantiate --eval --raw '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath)" > $ nix-copy-closure --from alice@itchy.example.org "$storePath" > $ "$storePath"/bin/hello > Hello, world! diff --git a/doc/manual/source/command-ref/nix-instantiate.md b/doc/manual/source/command-ref/nix-instantiate.md index 6f6fcdc1f..487ef8f10 100644 --- a/doc/manual/source/command-ref/nix-instantiate.md +++ b/doc/manual/source/command-ref/nix-instantiate.md @@ -5,7 +5,7 @@ # Synopsis `nix-instantiate` - [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ] + [`--parse` | `--eval` [`--strict`] [`--raw` | `--json` | `--xml`] ] [`--read-write-mode`] [`--arg` *name* *value*] [{`--attr`| `-A`} *attrPath*] @@ -102,6 +102,11 @@ standard input. > This option can cause non-termination, because lazy data > structures can be infinitely large. +- `--raw` + + When used with `--eval`, the evaluation result must be a string, + which is printed verbatim, without quoting, escaping or trailing newline. + - `--json` When used with `--eval`, print the resulting value as an JSON diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c48549511..09d354832 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -24,7 +24,7 @@ static Path gcRoot; static int rootNr = 0; -enum OutputKind { okPlain, okXML, okJSON }; +enum OutputKind { okPlain, okRaw, okXML, okJSON }; void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, @@ -50,7 +50,11 @@ void processExpr(EvalState & state, const Strings & attrPaths, vRes = v; else state.autoCallFunction(autoArgs, v, vRes); - if (output == okXML) + if (output == okRaw) + std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline + // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. + else if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); @@ -132,6 +136,8 @@ static int main_nix_instantiate(int argc, char * * argv) gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") ; + else if (*arg == "--raw") + outputKind = okRaw; else if (*arg == "--xml") outputKind = okXML; else if (*arg == "--json") diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 18f8589df..ed9c214f5 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -29,6 +29,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] [[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix-instantiate -A str --raw --eval "./eval.nix") == $'foo\nbar' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] From 5e21bdc623939a5b44f679f1436cb9e9e5d6eef5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 31 Dec 2024 17:24:36 +0100 Subject: [PATCH 179/243] .github/ci: Use fixed names This lets us update "runs-on" without creating a mismatch with the required checks that are configured for the repo in the github ui. --- .github/workflows/ci.yml | 28 +++++++++++++++++++++------- .mergify.yml | 8 ++++---- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd567cd63..6169c0924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-14] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: tests ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -37,7 +44,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'linux' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,8 +58,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-14] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: installer test ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 - name: Download installer tarball @@ -68,9 +82,9 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'linux' - run: brew install fish - if: matrix.os == 'macos-14' + if: matrix.os == 'darwin' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" diff --git a/.mergify.yml b/.mergify.yml index 13d2002ea..bed17f516 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,10 +2,10 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=tests (macos-14) - - check-success=tests (ubuntu-24.04) - - check-success=installer_test (macos-14) - - check-success=installer_test (ubuntu-24.04) + - check-success=tests on macos + - check-success=tests on ubuntu + - check-success=installer_test on macos + - check-success=installer_test on ubuntu - check-success=vm_tests batch_size: 5 From 4a2310a3a09a4e9bde51b003c2a562a4a49cfabf Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Tue, 31 Dec 2024 17:52:06 +0100 Subject: [PATCH 180/243] toJSON: re-throw serialization exception --- src/libexpr/value-to-json.cc | 6 +++++- src/libexpr/value-to-json.hh | 3 +++ tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp | 8 ++++++++ tests/functional/lang/eval-fail-toJSON-non-utf-8.nix | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp create mode 100644 tests/functional/lang/eval-fail-toJSON-non-utf-8.nix diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 8044fe347..5aa4fe4fd 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { - str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + try { + str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + } catch (nlohmann::json::exception & e) { + throw JSONSerializationError("JSON serialization error: %s", e.what()); + } } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 47ac90313..867c4e3a8 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); + +MakeError(JSONSerializationError, Error); + } diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp new file mode 100644 index 000000000..129d58bcb --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'toJSON' builtin + at /pwd/lang/eval-fail-toJSON-non-utf-8.nix:1:1: + 1| builtins.toJSON "_invalid UTF-8: _" + | ^ + 2| + + error: JSON serialization error: [json.exception.type_error.316] invalid UTF-8 byte at index 16: 0xFF diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix new file mode 100644 index 000000000..bd1f74de7 --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix @@ -0,0 +1 @@ +builtins.toJSON "_invalid UTF-8: _" From 01c96f9fd587912a50e746309580f75e95c56a97 Mon Sep 17 00:00:00 2001 From: Mutsuha Asada Date: Thu, 2 Jan 2025 13:26:13 +0900 Subject: [PATCH 181/243] libmain: fix ignoring empty lines in the print-build-logs option --- src/libmain/progress-bar.cc | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index fa0b73ebe..961850b58 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -287,23 +287,21 @@ public: else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = chomp(getS(fields, 0)); - if (!lastLine.empty()) { - auto i = state->its.find(act); - assert(i != state->its.end()); - ActInfo info = *i->second; - if (printBuildLogs) { - auto suffix = "> "; - if (type == resPostBuildLogLine) { - suffix = " (post)> "; - } - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); - } else { - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(*state); + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); } } From 359a0840e26fed25a2e978f6b3035436948430e6 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Sat, 21 Dec 2024 04:22:27 +0000 Subject: [PATCH 182/243] packaging: use optimization level 3 and LTO by default --- nix-meson-build-support/common/meson.build | 4 ---- packaging/dependencies.nix | 15 +++++++++++++++ src/libcmd/meson.build | 2 -- src/libexpr-c/meson.build | 2 -- src/libexpr-test-support/meson.build | 2 -- src/libexpr-tests/meson.build | 2 -- src/libexpr/meson.build | 2 -- src/libfetchers-tests/meson.build | 2 -- src/libfetchers/meson.build | 2 -- src/libflake-c/meson.build | 2 -- src/libflake-tests/meson.build | 2 -- src/libflake/meson.build | 2 -- src/libmain-c/meson.build | 2 -- src/libmain/meson.build | 2 -- src/libstore-c/meson.build | 2 -- src/libstore-test-support/meson.build | 2 -- src/libstore-tests/meson.build | 2 -- src/libstore/meson.build | 2 -- src/libutil-c/meson.build | 2 -- src/libutil-test-support/meson.build | 2 -- src/libutil-tests/meson.build | 2 -- src/libutil/meson.build | 2 -- src/nix/meson.build | 2 -- tests/functional/meson.build | 2 -- 24 files changed, 15 insertions(+), 48 deletions(-) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index f0322183e..67b6658f5 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -16,7 +16,3 @@ add_project_arguments( '-Wno-deprecated-declarations', language : 'cpp', ) - -if get_option('buildtype') not in ['debug'] - add_project_arguments('-O3', language : 'cpp') -endif diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2c7cf701f..4bc7495e7 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -66,6 +66,21 @@ let mesonLayer = finalAttrs: prevAttrs: { + # NOTE: + # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, + # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default. + # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype. + mesonBuildType = "release"; + # NOTE: + # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the + # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. + # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. + preConfigure = prevAttrs.preConfigure or "" + '' + case "$mesonBuildType" in + release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; + *) appendToVar mesonFlags "-Db_lto=false" ;; + esac + ''; nativeBuildInputs = [ pkgs.buildPackages.meson pkgs.buildPackages.ninja diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 914db0108..4145f408a 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -4,8 +4,6 @@ project('nix-cmd', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 1556dae51..9487132cf 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -4,8 +4,6 @@ project('nix-expr-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 64d4fe218..56e814cd1 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-expr-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index f37e85e57..667a0d7b7 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -4,8 +4,6 @@ project('nix-expr-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index b3c559ba7..b33aebc86 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -4,8 +4,6 @@ project('nix-expr', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 3e82c6111..739435501 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index b4408e943..58afbb7d0 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index b7669fe97..85d20644d 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -4,8 +4,6 @@ project('nix-flake-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 5c3c58e53..1c8765f21 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -4,8 +4,6 @@ project('nix-flake-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake/meson.build b/src/libflake/meson.build index f9e217729..b757d0d76 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -4,8 +4,6 @@ project('nix-flake', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 5a5684b8d..d875d2c3f 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -4,8 +4,6 @@ project('nix-main-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 87fc8b8d2..00f945f49 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -4,8 +4,6 @@ project('nix-main', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 1ac331ad0..17d18609f 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -4,8 +4,6 @@ project('nix-store-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 2a07e56ac..59d649889 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-store-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index b706fa12c..3ba0795e9 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -4,8 +4,6 @@ project('nix-store-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 12a0e6376..79d912497 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -4,8 +4,6 @@ project('nix-store', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 44cec1afc..ac1297665 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -4,8 +4,6 @@ project('nix-util-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 03ae63f1a..db944cf06 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-util-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 83ac79e92..ad2c61711 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -4,8 +4,6 @@ project('nix-util-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 2c3e3a954..ac701d8fd 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -4,8 +4,6 @@ project('nix-util', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/nix/meson.build b/src/nix/meson.build index 1d4840b12..2698cc873 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -4,8 +4,6 @@ project('nix', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 933595cd5..83e08c4f5 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -4,8 +4,6 @@ project('nix-functional-tests', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.3', From a44e9dd1ea664a9616d7dabb548095bb3f375f7e Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 4 Jan 2025 16:14:06 +0100 Subject: [PATCH 183/243] correctly parse strings with null bytes and throw error --- src/libexpr/eval.cc | 8 ++++-- src/libexpr/lexer.l | 25 +++++++++++------- src/libexpr/value.hh | 2 +- .../lang/eval-fail-string-nul-1.err.exp | Bin 0 -> 209 bytes .../lang/eval-fail-string-nul-1.nix | Bin 0 -> 10 bytes .../lang/eval-fail-string-nul-2.err.exp | Bin 0 -> 256 bytes .../lang/eval-fail-string-nul-2.nix | Bin 0 -> 22 bytes 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 tests/functional/lang/eval-fail-string-nul-1.err.exp create mode 100644 tests/functional/lang/eval-fail-string-nul-1.nix create mode 100644 tests/functional/lang/eval-fail-string-nul-2.err.exp create mode 100644 tests/functional/lang/eval-fail-string-nul-2.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fe5f05ab8..21dd5a294 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3185,12 +3185,16 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } -void forceNoNullByte(std::string_view s) +void forceNoNullByte(std::string_view s, std::function pos) { if (s.find('\0') != s.npos) { using namespace std::string_view_literals; auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); - throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + Error error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + if (pos) { + error.atPos(pos()); + } + throw error; } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a7e44cb72..067f86e01 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -41,16 +41,18 @@ namespace nix { // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. -static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) +// getting the position is expensive and thus it is implemented lazily. +static StringToken unescapeStr(char * const s, size_t length, std::function && pos) { - char * result = s; + bool noNullByte = true; char * t = s; - char c; // the input string is terminated with *two* NULs, so we can safely take // *one* character after the one being checked against. - while ((c = *s++)) { + for (size_t i = 0; i < length; t++) { + char c = s[i++]; + noNullByte &= c != '\0'; if (c == '\\') { - c = *s++; + c = s[i++]; if (c == 'n') *t = '\n'; else if (c == 'r') *t = '\r'; else if (c == 't') *t = '\t'; @@ -59,12 +61,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) else if (c == '\r') { /* Normalise CR and CR/LF into LF. */ *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ + if (s[i] == '\n') i++; /* cr/lf */ } else *t = c; - t++; } - return {result, size_t(t - result)}; + if (!noNullByte) { + forceNoNullByte({s, size_t(t - s)}, std::move(pos)); + } + return {s, size_t(t - s)}; } static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) @@ -175,7 +179,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(state->symbols, yytext, yyleng); + yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -191,6 +195,7 @@ or { return OR_KW; } \'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } ([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->str = {yytext, (size_t) yyleng, true}; + forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \'\'\$ | @@ -203,7 +208,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 88fcae986..8925693e3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,6 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); -void forceNoNullByte(std::string_view s); +void forceNoNullByte(std::string_view s, std::function = nullptr); } diff --git a/tests/functional/lang/eval-fail-string-nul-1.err.exp b/tests/functional/lang/eval-fail-string-nul-1.err.exp new file mode 100644 index 0000000000000000000000000000000000000000..2dfbea0635c8c2ea159249879c4d89523c2a7366 GIT binary patch literal 209 zcmYc-D#|aiQpn6JC@oPaE-A{)OIJ`&%g=u_p&==;NL?X0F)uH_L?J0vp(wSWD782> zuOu}^A+cD&FS7!sJ}EUhv9vfR(-JdtbRl-?=9T8?8tUa`R#+KY8Ck*ffOQ(yC@7`n=QAWF7Ac`f PDAXt@#G!~7)o=j-ygENH literal 0 HcmV?d00001 diff --git a/tests/functional/lang/eval-fail-string-nul-1.nix b/tests/functional/lang/eval-fail-string-nul-1.nix new file mode 100644 index 0000000000000000000000000000000000000000..3689409171139a7d310eac52df0be4315b6ba786 GIT binary patch literal 10 RcmY#N%g<*>N-R?10ss()0>}UW literal 0 HcmV?d00001 diff --git a/tests/functional/lang/eval-fail-string-nul-2.err.exp b/tests/functional/lang/eval-fail-string-nul-2.err.exp new file mode 100644 index 0000000000000000000000000000000000000000..b1cae5325d90a2c05f7304db89bca0826c2f98f0 GIT binary patch literal 256 zcmYc-D#|aiQpn6JC@oPaE-A{)OIJ`=P)N(q=Td-yM-v(loTS7e7)M Date: Sun, 5 Jan 2025 19:02:28 +0000 Subject: [PATCH 184/243] windows: fix conditional compilation variable --- src/libstore/windows/pathlocks.cc | 2 +- src/libutil/windows/environment-variables.cc | 2 +- src/libutil/windows/file-descriptor.cc | 2 +- src/libutil/windows/file-system.cc | 2 +- src/libutil/windows/muxable-pipe.cc | 2 +- src/libutil/windows/os-string.cc | 2 +- src/libutil/windows/processes.cc | 2 +- src/libutil/windows/users.cc | 2 +- src/libutil/windows/windows-async-pipe.cc | 2 +- src/libutil/windows/windows-async-pipe.hh | 2 +- src/libutil/windows/windows-error.cc | 2 +- src/libutil/windows/windows-error.hh | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 197f5a1c4..29a98d8e2 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -3,7 +3,7 @@ #include "signals.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 # include # include # include diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 308a432e4..d1093597c 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,6 @@ #include "environment-variables.hh" -#ifdef WIN32 +#ifdef _WIN32 # include "processenv.h" namespace nix { diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 71f53ccb8..e2a473a7c 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,7 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #include #include diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 53271cef3..7ed1c04a6 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,6 +1,6 @@ #include "file-system.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index af7e987e9..ac2882120 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,4 +1,4 @@ -#ifdef WIN32 +#ifdef _WIN32 # include # include "windows-error.hh" diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 26ad9cba0..b09ef8b90 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,7 +7,7 @@ #include "file-path-impl.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index e69f1ed45..fd4d7c43a 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,7 +23,7 @@ #include #include -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index 2780e45f4..438c4221c 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,7 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 13b563510..4e139d5cf 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,7 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 277336ed7..53715e260 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,7 +2,7 @@ ///@file #include "file-descriptor.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 4cf4274da..b92f9155f 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,6 +1,6 @@ #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 4e48ee859..66c67b43a 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#ifdef WIN32 +#ifdef _WIN32 #include #include "error.hh" From 1eba904b7994985996f6ee1571ef70c6e9a91ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 5 Jan 2025 19:14:29 +0000 Subject: [PATCH 185/243] nix/flake: fix build on windows --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 55aa8971e..7189689cc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -938,7 +938,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - createSymlink(target, to2); + createSymlink(target, os_string_to_string(PathViewNG { to2 })); } else throw Error("file '%s' has unsupported type", from2); From 438a20427f40e8ec458444fb156e984f8137ec13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 6 Jan 2025 22:53:39 +0100 Subject: [PATCH 186/243] mergify: fix installer test name --- .mergify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index bed17f516..5d2bf8520 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,8 +4,8 @@ queue_rules: merge_conditions: - check-success=tests on macos - check-success=tests on ubuntu - - check-success=installer_test on macos - - check-success=installer_test on ubuntu + - check-success=installer test on macos + - check-success=installer test on ubuntu - check-success=vm_tests batch_size: 5 From 69853c067c74e9421cfa41e48bf8af04266206d3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:37:25 +0100 Subject: [PATCH 187/243] Add makeParentCanonical() --- src/libutil/file-system.cc | 15 +++++++++++++++ src/libutil/file-system.hh | 17 +++++++++++++++++ src/libutil/posix-source-accessor.hh | 12 ++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 793eb2d9f..923220fd0 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -766,4 +766,19 @@ bool isExecutableFileAmbient(const fs::path & exe) { ) == 0; } +std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) +{ + std::filesystem::path path(absPath(rawPath));; + try { + auto parent = path.parent_path(); + if (parent == path) { + // `path` is a root directory => trivially canonical + return parent; + } + return std::filesystem::canonical(parent) / path.filename(); + } catch (fs::filesystem_error & e) { + throw SysError("canonicalising parent path of '%1%'", path); + } } + +} // namespace nix diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 3c49181a0..7fdaba811 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -143,6 +143,23 @@ inline bool symlink_exists(const std::filesystem::path & path) { } // namespace fs +/** + * Canonicalize a path except for the last component. + * + * This is useful for getting the canonical location of a symlink. + * + * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will + * resolve the symlink `l` to its target. + * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, + * but does ensure that the returned parent part of the path, `foo` is resolved + * to `canonical("foo")`, and can therefore be retrieved without traversing any + * symlinks. + * + * If a relative path is passed, it will be made absolute, so that the parent + * can always be canonicalized. + */ +std::filesystem::path makeParentCanonical(const std::filesystem::path & path); + /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 40f60bb54..d60abdd9a 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -50,6 +50,18 @@ struct PosixSourceAccessor : virtual SourceAccessor * possible, (e.g. on Windows it could scoped to a drive like * `C:\`). This allows more `..` parent accessing to work. * + * @note When `path` is trusted user input, canonicalize it using + * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, + * as appropriate for the use case. At least weak canonicalization is + * required for the `SourcePath` to do anything useful at the location it + * points to. + * + * @note A canonicalizing behavior is not built in `createAtRoot` so that + * callers do not accidentally introduce symlink-related security vulnerabilities. + * Furthermore, `createAtRoot` does not know whether the file pointed to by + * `path` should be resolved if it is itself a symlink. In other words, + * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. + * * See * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) * and From 36563c69a48a47d8b58340f22d82d542f6b9f025 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 17:21:56 +0100 Subject: [PATCH 188/243] fix: Handle symlinks and FIFOs in `nix hash` where possible Fixes https://github.com/NixOS/nix/issues/11756 Fixes https://github.com/NixOS/nix/issues/11681 --- src/libutil/posix-source-accessor.hh | 2 +- src/nix/hash.cc | 23 ++++++++++++++++++++--- tests/functional/hash-path.sh | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index d60abdd9a..5d491e633 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -43,7 +43,7 @@ struct PosixSourceAccessor : virtual SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; /** - * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * Create a `PosixSourceAccessor` and `SourcePath` corresponding to * some native path. * * The `PosixSourceAccessor` is rooted as far up the tree as diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 416cd19b3..eac421d12 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -87,18 +87,35 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto path2 = PosixSourceAccessor::createAtRoot(path); + auto makeSourcePath = [&]() -> SourcePath { + return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: + { + // While usually we could use the some code as for NixArchive, + // the Flat method needs to support FIFOs, such as those + // produced by bash process substitution, e.g.: + // nix hash --mode flat <(echo hi) + // Also symlinks semantics are unambiguous in the flat case, + // so we don't need to go low-level, or reject symlink `path`s. + auto hashSink = makeSink(); + readFile(path, *hashSink); + h = hashSink->finish().first; + break; + } case FileIngestionMethod::NixArchive: { + auto sourcePath = makeSourcePath(); auto hashSink = makeSink(); - dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); + dumpPath(sourcePath, *hashSink, (FileSerialisationMethod) mode); h = hashSink->finish().first; break; } case FileIngestionMethod::Git: { + auto sourcePath = makeSourcePath(); std::function hook; hook = [&](const SourcePath & path) -> git::TreeEntry { auto hashSink = makeSink(); @@ -109,7 +126,7 @@ struct CmdHashBase : Command .hash = hash, }; }; - h = hook(path2).hash; + h = hook(sourcePath).hash; break; } } diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 86d782a95..74e319504 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -92,3 +92,27 @@ try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" rm "$TEST_ROOT/hash-path/hello" ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" + +# Flat mode supports process substitution +h=$(nix hash path --mode flat --type sha256 --base32 <(printf "SMASH THE STATE")) +[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]] + +# Flat mode supports process substitution (hash file) +h=$(nix hash file --type sha256 --base32 <(printf "SMASH THE STATE")) +[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]] + +# Symlinks in the ancestry are ok and don't affect the result +mkdir -p "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it" +echo hi > "$TEST_ROOT/simple/hi" +ln -s "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it/simple-link" +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/simple/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple-link/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] + +# nix hash --mode nar does not canonicalize a symlink argument. +# Otherwise it can't generate a NAR whose root is a symlink. +# If you want to follow the symlink, pass $(realpath -s ...) instead. +ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere" +h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere") +[[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]] # manually verified hash From 91e91f62fa2187f49073851f9a3dbff740092859 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:38:47 +0100 Subject: [PATCH 189/243] doc: Document `nix-store --add-fixed` symlink behavior Tested with nix run nix/2.3-maintenance#nix-store -- --add some_symlink nix run nix/2.3-maintenance#nix-store -- --add-fixed sha256 --recursive some_symlink --- doc/manual/source/command-ref/nix-store/add-fixed.md | 3 +++ doc/manual/source/command-ref/nix-store/add.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/doc/manual/source/command-ref/nix-store/add-fixed.md b/doc/manual/source/command-ref/nix-store/add-fixed.md index bebf15026..2ea90a135 100644 --- a/doc/manual/source/command-ref/nix-store/add-fixed.md +++ b/doc/manual/source/command-ref/nix-store/add-fixed.md @@ -21,6 +21,9 @@ This operation has the following options: Use recursive instead of flat hashing mode, used when adding directories to the store. + *paths* that refer to symlinks are not dereferenced, but added to the store + as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/source/command-ref/nix-store/add.md b/doc/manual/source/command-ref/nix-store/add.md index 87d504cd3..ab4740723 100644 --- a/doc/manual/source/command-ref/nix-store/add.md +++ b/doc/manual/source/command-ref/nix-store/add.md @@ -11,6 +11,9 @@ The operation `--add` adds the specified paths to the Nix store. It prints the resulting paths in the Nix store on standard output. +*paths* that refer to symlinks are not dereferenced, but added to the store +as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} From c0b64f3377b37c036c574b731c3aeffb3fddc614 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:46:47 +0100 Subject: [PATCH 190/243] refactor: Don't re-construct SourcePath unnecessarily --- src/nix-store/nix-store.cc | 8 ++++---- src/nix/add-to-store.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b731b25af..659a237f9 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -183,9 +183,9 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(i); cout << fmt("%s\n", store->printStorePath(store->addToStore( - std::string(baseNameOf(i)), {accessor, canonPath}))); + std::string(baseNameOf(i)), sourcePath))); } } @@ -207,10 +207,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(i); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), - {accessor, canonPath}, + sourcePath, method, hashAlgo).path)); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5c08f7616..45563cb2b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -37,13 +37,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); + auto sourcePath = PosixSourceAccessor::createAtRoot(path); auto storePath = dryRun ? store->computeStorePath( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).first + *namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).path; + *namePart, sourcePath, caMethod, hashAlgo, {}).path; logger->cout("%s", store->printStorePath(storePath)); } From ddbbf537676441ff5c09bd64310fec0f2619a820 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:54:36 +0100 Subject: [PATCH 191/243] fix: Resolve CLI parent symlinks before adding to store Fixes https://github.com/NixOS/nix/issues/11941 --- src/nix-store/nix-store.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- tests/functional/add.sh | 41 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 659a237f9..99bb2c726 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -183,7 +183,7 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); cout << fmt("%s\n", store->printStorePath(store->addToStore( std::string(baseNameOf(i)), sourcePath))); } @@ -207,7 +207,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), sourcePath, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 45563cb2b..7f15de374 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -37,7 +37,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto sourcePath = PosixSourceAccessor::createAtRoot(path); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); auto storePath = dryRun ? store->computeStorePath( diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 3b37ee7d4..0e6868d8f 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -29,6 +29,47 @@ echo "$hash2" test "$hash1" = "sha256:$hash2" +# The contents can be accessed through a symlink, and this symlink has no effect on the hash +# https://github.com/NixOS/nix/issues/11941 +test_issue_11941() { + local expected actual + mkdir -p "$TEST_ROOT/foo/bar" && ln -s "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" + + # legacy + expected=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo/bar") + actual=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + actual=$(nix-store --add "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + + # nix store add + actual=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/foo/bar") + [[ "$expected" == "$actual" ]] + + # cleanup + rm -r "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" +} +test_issue_11941 + +# A symlink is added to the store as a symlink, not as a copy of the target +test_add_symlink() { + ln -s /bin "$TEST_ROOT/my-bin" + + # legacy + path=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + path=$(nix-store --add "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # nix store add + path=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # cleanup + rm "$TEST_ROOT/my-bin" +} +test_add_symlink + #### New style commands clearStoreIfPossible From 628c11d237dc38bf4e35c498d418026c215de5cf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 16 Dec 2024 09:53:36 +0100 Subject: [PATCH 192/243] test: Add hydraJobs.tests.functional_symlinked-home --- tests/nixos/default.nix | 2 ++ tests/nixos/functional/symlinked-home.nix | 36 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/nixos/functional/symlinked-home.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index a9a438a32..8e0cb1b22 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -159,6 +159,8 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + functional_symlinked-home = runNixOSTestFor "x86_64-linux" ./functional/symlinked-home.nix; + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix new file mode 100644 index 000000000..57c45d5d5 --- /dev/null +++ b/tests/nixos/functional/symlinked-home.nix @@ -0,0 +1,36 @@ +/** + This test runs the functional tests on a NixOS system where the home directory + is symlinked to another location. + + The purpose of this test is to find cases where Nix uses low-level operations + that don't support symlinks on paths that include them. + + It is not a substitute for more intricate, use case-specific tests, but helps + catch common issues. +*/ +# TODO: add symlinked tmpdir +{ ... }: +{ + name = "functional-tests-on-nixos_user_symlinked-home"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + with subtest("prepare symlinked home"): + machine.succeed(""" + ( + set -x + mv /home/alice /home/alice.real + ln -s alice.real /home/alice + ) 1>&2 + """) + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} From 5a5a86949a2fca18f61d1117ab29cc87281c48d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 19:03:14 +0100 Subject: [PATCH 193/243] makeParentCanonical: test case where parent is empty --- src/libutil-tests/file-system.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 7ef804f34..2c10d4869 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -261,4 +261,18 @@ TEST(pathExists, bogusPathDoesNotExist) { ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } + +/* ---------------------------------------------------------------------------- + * makeParentCanonical + * --------------------------------------------------------------------------*/ + +TEST(makeParentCanonical, noParent) +{ + ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); +} + +TEST(makeParentCanonical, root) +{ + ASSERT_EQ(makeParentCanonical("/"), "/"); +} } From 4c74d679b6b4db9eccea94afc285176871253967 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Dec 2024 16:55:27 +0100 Subject: [PATCH 194/243] test: Avoid regressing accidental use of weakly_canonical instead of makeParentCanonical I'd messed up a rebase in my previous iteration, causing `weakly_canonical` to reappear, but not trigger a test failure. These two functions behave similarly when the argument is a path that points to a broken symlink. `weakly_canonical` would not resolve it because the target doesn't exist, and `makeParentCanonical` would not resolve it, because it never resolves the final path element. This new test case now also tests a valid symlink, "differentiating" the two. --- tests/functional/hash-path.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 74e319504..4894ae391 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -116,3 +116,8 @@ h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple- ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere" h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere") [[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]] # manually verified hash +if [[ -e /bin ]]; then + ln -s /bin "$TEST_ROOT/symlink-to-bin" + h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-bin") + [[ 0z2mdmkd43l0ijdxfbj1y8vzli15yh9b09n3a3rrygmjshbyypsw == "$h" ]] # manually verified hash +fi From 75cda2da7fbd6283f88cf0fd0b934e1047ca1408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:40:18 +0100 Subject: [PATCH 195/243] Document path values in inputs Co-authored-by: Robert Hensing --- src/nix/flake.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index a55bba4fc..5bb18328d 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -212,6 +212,13 @@ Currently the `type` attribute can be one of the following: root of a tree can use `path:./foo` to access the flake in subdirectory `foo`, but `path:../bar` is illegal. +Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by +1. resolving them into relative paths, relative to the base directory of `flake.nix` +2. escaping URL characters (refer to IETF RFC?) +3. prepending `path:` + +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) + Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). From e8c7dd9971aaac0ac14e08f299a41ef0a93afef7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:43:56 +0100 Subject: [PATCH 196/243] Rename allowRelative -> preserveRelativePaths --- src/libflake/flake/flakeref.cc | 12 ++++++------ src/libflake/flake/flakeref.hh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 3a43a51f8..06198b10c 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -49,9 +49,9 @@ FlakeRef parseFlakeRef( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -89,7 +89,7 @@ std::pair parsePathFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { std::string path = url; std::string fragment = ""; @@ -191,7 +191,7 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!allowRelative && !hasPrefix(path, "/")) + if (!preserveRelativePaths && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); } @@ -258,7 +258,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { using namespace fetchers; @@ -267,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); } } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 32094d381..c9cf7952d 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -85,7 +85,7 @@ FlakeRef parseFlakeRef( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -104,7 +104,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) From 0792152627a94dbb2fbbbd8f0f05ff36a480dabf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:54:19 +0100 Subject: [PATCH 197/243] Rename Override -> OverrideTarget --- src/libflake/flake/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e9fa28f06..4f0b922ab 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -408,14 +408,14 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - struct Override + struct OverrideTarget { FlakeInput input; SourcePath sourcePath; std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; - std::map overrides; + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; @@ -423,7 +423,7 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputOverrides) { overrides.emplace( i.first, - Override { + OverrideTarget { .input = FlakeInput { .ref = i.second }, /* Note: any relative overrides (e.g. `--override-input B/C "path:./foo/bar"`) @@ -474,7 +474,7 @@ LockedFlake lockFlake( inputPath.push_back(id); inputPath.push_back(idOverride); overrides.emplace(inputPath, - Override { + OverrideTarget { .input = inputOverride, .sourcePath = sourcePath, .parentInputPath = inputPathPrefix From ef2739b7c9071149fb9333865bd3cc2ab60f2178 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:01:49 +0100 Subject: [PATCH 198/243] Example of referencing parent directories --- src/nix/flake.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 5bb18328d..35ac3ea9b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -210,7 +210,9 @@ Currently the `type` attribute can be one of the following: the directory containing that `flake.nix`. However, the resolved path must be in the same tree. For instance, a `flake.nix` in the root of a tree can use `path:./foo` to access the flake in - subdirectory `foo`, but `path:../bar` is illegal. + subdirectory `foo`, but `path:../bar` is illegal. On the other + hand, a flake in the `/foo` directory of a tree can use + `path:../bar` to refer to the flake in `/bar`. Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by 1. resolving them into relative paths, relative to the base directory of `flake.nix` From f705ce7f9a95bc37df236d88eef1dbd9e2ae5f31 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:45:58 +0100 Subject: [PATCH 199/243] ParsedURL: Remove url field This prevents a 'url' field that is out of sync with the other fields. You can use to_string() to get the full URL. --- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/github.cc | 16 ++++++++-------- src/libfetchers/indirect.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libflake/flake/flakeref.cc | 2 -- src/libutil-tests/url.cc | 22 ---------------------- src/libutil/url.cc | 7 ++++++- src/libutil/url.hh | 5 +++-- 8 files changed, 25 insertions(+), 43 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b105c252a..87bf0dd71 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -66,7 +66,7 @@ Input Input::fromURL( } } - throw Error("input '%s' is unsupported", url.url); + throw Error("input '%s' is unsupported", url); } Input Input::fromAttrs(const Settings & settings, Attrs && attrs) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 308cff33a..185941988 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -50,7 +50,7 @@ struct GitArchiveInputScheme : InputScheme else if (std::regex_match(path[2], refRegex)) ref = path[2]; else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -63,34 +63,34 @@ struct GitArchiveInputScheme : InputScheme if (std::regex_match(rs, refRegex)) { ref = rs; } else { - throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs); } } else if (size < 2) - throw BadURL("URL '%s' is invalid", url.url); + throw BadURL("URL '%s' is invalid", url); for (auto &[name, value] : url.query) { if (name == "rev") { if (rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); + throw BadURL("URL '%s' contains multiple commit hashes", url); rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + throw BadURL("URL '%s' contains an invalid branch/tag name", url); if (ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + throw BadURL("URL '%s' contains multiple branch/tag names", url); ref = value; } else if (name == "host") { if (!std::regex_match(value, hostRegex)) - throw BadURL("URL '%s' contains an invalid instance host", url.url); + throw BadURL("URL '%s' contains an invalid instance host", url); host_url = value; } // FIXME: barf on unsupported attributes } if (ref && rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev()); Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 2e5cd82c7..0e1b86711 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -26,16 +26,16 @@ struct IndirectInputScheme : InputScheme else if (std::regex_match(path[1], refRegex)) ref = path[1]; else - throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) - throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url, path[1]); ref = path[1]; if (!std::regex_match(path[2], revRegex)) - throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]); rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else - throw BadURL("GitHub URL '%s' is invalid", url.url); + throw BadURL("GitHub URL '%s' is invalid", url); std::string id = path[0]; if (!std::regex_match(id, flakeRegex)) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 246b68c3a..f628e042c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -14,7 +14,7 @@ struct PathInputScheme : InputScheme if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority); Input input{settings}; input.attrs.insert_or_assign("type", "path"); @@ -27,10 +27,10 @@ struct PathInputScheme : InputScheme if (auto n = string2Int(value)) input.attrs.insert_or_assign(name, *n); else - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has invalid parameter '%s'", url, name); } else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has unsupported parameter '%s'", url, name); return input; } diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index ab882fdab..957495795 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -162,7 +162,6 @@ std::pair parsePathFlakeRefWithFragment( auto base = std::string("git+file://") + flakeRoot; auto parsedURL = ParsedURL{ - .url = base, // FIXME .base = base, .scheme = "git+file", .authority = "", @@ -220,7 +219,6 @@ static std::optional> parseFlakeIdRef( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ - .url = url, .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 7d08f467e..6ffae4f3e 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -20,23 +20,11 @@ namespace nix { } - std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { - return os << "\n" - << "url: " << p.url << "\n" - << "base: " << p.base << "\n" - << "scheme: " << p.scheme << "\n" - << "authority: " << p.authority.value() << "\n" - << "path: " << p.path << "\n" - << "query: " << print_map(p.query) << "\n" - << "fragment: " << p.fragment << "\n"; - } - TEST(parseURL, parsesSimpleHttpUrl) { auto s = "http://www.example.org/file.tar.gz"; auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", @@ -53,7 +41,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", @@ -70,7 +57,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", @@ -87,7 +73,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", @@ -104,7 +89,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "file+https://www.example.org/video.mp4", .base = "https://www.example.org/video.mp4", .scheme = "file+https", .authority = "www.example.org", @@ -126,7 +110,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://127.0.0.1:8080/file.tar.gz", .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", .authority = "127.0.0.1:8080", @@ -143,7 +126,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", @@ -161,7 +143,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", @@ -185,7 +166,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://user:pass@www.example.org/file.tar.gz", .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", .authority = "user:pass@www.example.org:8080", @@ -203,7 +183,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "", .base = "", .scheme = "file", .authority = "", @@ -228,7 +207,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "ftp://ftp.nixos.org/downloads/nixos.iso", .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", .authority = "ftp.nixos.org", diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 63b9734ee..86ca035b9 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -40,7 +40,6 @@ ParsedURL parseURL(const std::string & url) path = "/"; return ParsedURL{ - .url = url, .base = base, .scheme = scheme, .authority = authority, @@ -136,6 +135,12 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } +std::ostream & operator << (std::ostream & os, const ParsedURL & url) +{ + os << url.to_string(); + return os; +} + bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 738ee9f82..d85001185 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -7,9 +7,8 @@ namespace nix { struct ParsedURL { - std::string url; /// URL without query/fragment - std::string base; + std::string base; // FIXME: remove std::string scheme; std::optional authority; std::string path; @@ -26,6 +25,8 @@ struct ParsedURL ParsedURL canonicalise(); }; +std::ostream & operator << (std::ostream & os, const ParsedURL & url); + MakeError(BadURL, Error); std::string percentDecode(std::string_view in); From 4077aa43a803db33108ce39e799089608707dcbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:52:00 +0100 Subject: [PATCH 200/243] ParsedURL: Remove base field --- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libflake/flake/flakeref.cc | 4 ---- src/libutil-tests/url.cc | 11 ----------- src/libutil/url.cc | 1 - src/libutil/url.hh | 2 -- 6 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c73f53765..d894550c0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -426,7 +426,7 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - repoInfo.url = repoInfo.isLocal ? url.path : url.base; + repoInfo.url = repoInfo.isLocal ? url.path : url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2c987f79d..c2fd8139c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -161,7 +161,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? url.path : url.base}; + return {isLocal, isLocal ? url.path : url.to_string()}; } StorePath fetchToStore(ref store, Input & input) const diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 957495795..60efe1612 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -159,10 +159,7 @@ std::pair parsePathFlakeRefWithFragment( while (flakeRoot != "/") { if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - auto parsedURL = ParsedURL{ - .base = base, .scheme = "git+file", .authority = "", .path = flakeRoot, @@ -219,7 +216,6 @@ static std::optional> parseFlakeIdRef( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ - .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", .path = match[1], diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 6ffae4f3e..7e1d2aa15 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -25,7 +25,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -41,7 +40,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -57,7 +55,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -73,7 +70,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -89,7 +85,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/video.mp4", .scheme = "file+https", .authority = "www.example.org", .path = "/video.mp4", @@ -110,7 +105,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", .authority = "127.0.0.1:8080", .path = "/file.tar.gz", @@ -126,7 +120,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .path = "", @@ -143,7 +136,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .path = "", @@ -166,7 +158,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", .authority = "user:pass@www.example.org:8080", .path = "/file.tar.gz", @@ -183,7 +174,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "", .scheme = "file", .authority = "", .path = "/none/of//your/business", @@ -207,7 +197,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", .authority = "ftp.nixos.org", .path = "/downloads/nixos.iso", diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 86ca035b9..4d4d983b8 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -40,7 +40,6 @@ ParsedURL parseURL(const std::string & url) path = "/"; return ParsedURL{ - .base = base, .scheme = scheme, .authority = authority, .path = percentDecode(path), diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d85001185..2b12f5af2 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -7,8 +7,6 @@ namespace nix { struct ParsedURL { - /// URL without query/fragment - std::string base; // FIXME: remove std::string scheme; std::optional authority; std::string path; From d329b2632a91b05e1151decbb2b4c4011ebf5aa1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 17:04:06 +0100 Subject: [PATCH 201/243] Fix manual --- src/nix/flake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 35ac3ea9b..e79a8ee0b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -219,7 +219,7 @@ Path inputs can be specified with path values in `flake.nix`. Path values are a 2. escaping URL characters (refer to IETF RFC?) 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From 3a5fccc41881a20c183038c02dea7f4995d4eabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20Mi=C5=A1kovi=C4=87?= <157494086+allrealmsoflife@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:42:26 +0100 Subject: [PATCH 202/243] outdated building instructions, update documentation.md The current instructions for building the Nix manual include a command that doesn't work as described. Specifically: ``` nix build .#nix^doc ``` Running this command results in the error: ``` error: derivation '/nix/store/hddqxzfqgx2fhj8q66ss3idym7pk7aj1-nix-2.26.0pre20250107_383ab87.drv' does not have wanted outputs 'doc' ``` However, this command works if you specify the Nix version explicitly, such as: ``` nix build nix/2.24.11#nix^doc ``` Additionally, these commands are run within the Nix root directory. However, the nix build .#nix^doc command does work when run from the nixpkgs directory and generates the NixOS manual. I'm not sure if I'm missing something. Is the `nix^doc` supposed to be added somehow to flake outputs? The incremental build section does not work since as make has been decommissioned in favor of Meson. Should this be simply deleted? --- doc/manual/source/development/documentation.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/development/documentation.md b/doc/manual/source/development/documentation.md index 2e188f232..30cc8adc4 100644 --- a/doc/manual/source/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -19,10 +19,11 @@ nix-build -E '(import ./.).packages.${builtins.currentSystem}.nix.doc' or ```console -nix build .#nix^doc +nix build .#nix-manual ``` -and open `./result-doc/share/doc/nix/manual/index.html`. +and open `./result/share/doc/nix/manual/index.html`. + To build the manual incrementally, [enter the development shell](./building.md) and run: From 5230d3ecc4cd3a3d965902a56b5a21bcc99821c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20Mi=C5=A1kovi=C4=87?= <157494086+allrealmsoflife@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:20:44 +0100 Subject: [PATCH 203/243] Document `--max-freed` for `nix-collect-garbage` (#12155) * Update nix-collect-garbage.md Referencing issue at: https://github.com/NixOS/nix/issues/12132 Copied the description of `--max-freed` option from https://github.com/NixOS/nix/blob/442a2623e48357ff72c77bb11cf2cf06d94d2f90/doc/manual/source/command-ref/nix-store/gc.md?plain=1#L39-L44 --- doc/manual/source/command-ref/nix-collect-garbage.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index bd05f2816..763179b8e 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,6 +62,15 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. + - [`--max-freed`](#opt-max-freed) *bytes* + + + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + {{#include ./opt-common.md}} {{#include ./env-common.md}} From ed4f2c32040afcc7f1e5adb89b67b3f28e384f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 7 Jan 2025 21:47:55 +0100 Subject: [PATCH 204/243] scripts/install-darwin-multi-user: workaround dscl failing sometimes --- scripts/install-darwin-multi-user.sh | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 89c66b8f4..c74999512 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -145,13 +145,28 @@ poly_user_id_get() { dsclattr "/Users/$1" "UniqueID" } +dscl_create() { + # workaround a bug in dscl where it sometimes fails with eNotYetImplemented: + # https://github.com/NixOS/nix/issues/12140 + while ! _sudo "$1" /usr/bin/dscl . -create "$2" "$3" "$4" 2> "$SCRATCH/dscl.err"; do + local err=$? + if [[ $err -eq 140 ]] && grep -q "-14988 (eNotYetImplemented)" "$SCRATCH/dscl.err"; then + echo "dscl failed with eNotYetImplemented, retrying..." + sleep 1 + continue + fi + cat "$SCRATCH/dscl.err" + return $err + done +} + poly_user_hidden_get() { dsclattr "/Users/$1" "IsHidden" } poly_user_hidden_set() { - _sudo "in order to make $1 a hidden user" \ - /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" + dscl_create "in order to make $1 a hidden user" \ + "/Users/$1" "IsHidden" "1" } poly_user_home_get() { @@ -161,8 +176,8 @@ poly_user_home_get() { poly_user_home_set() { # This can trigger a permission prompt now: # "Terminal" would like to administer your computer. Administration can include modifying passwords, networking, and system settings. - _sudo "in order to give $1 a safe home directory" \ - /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" + dscl_create "in order to give $1 a safe home directory" \ + "/Users/$1" "NFSHomeDirectory" "$2" } poly_user_note_get() { @@ -170,8 +185,8 @@ poly_user_note_get() { } poly_user_note_set() { - _sudo "in order to give $username a useful note" \ - /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" + dscl_create "in order to give $1 a useful note" \ + "/Users/$1" "RealName" "$2" } poly_user_shell_get() { @@ -179,8 +194,8 @@ poly_user_shell_get() { } poly_user_shell_set() { - _sudo "in order to give $1 a safe shell" \ - /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" + dscl_create "in order to give $1 a safe shell" \ + "/Users/$1" "UserShell" "$2" } poly_user_in_group_check() { From 28caa35a97ed4837a62cb054f66cf02041292cc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jan 2025 18:38:53 +0100 Subject: [PATCH 205/243] parsePathFlakeRefWithFragment(): Handle 'path?query' without a fragment Commands like `nix flake metadata '.?submodules=1'` ignored the query part of the URL, while `nix build '.?submodules=1#foo'` did work correctly because of the presence of the fragment part. --- src/libflake/flake/flakeref.cc | 27 +++++++------------ tests/functional/flakes/flake-in-submodule.sh | 13 +++++++++ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 60efe1612..58cd45632 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -89,23 +89,16 @@ std::pair parsePathFlakeRefWithFragment( bool allowMissing, bool isFlake) { - std::string path = url; - std::string fragment = ""; - std::map query; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') { - fragmentStart = url.find("#"); - } - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { - query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); - } + static std::regex pathFlakeRegex( + R"(([^?#]*)(\?([^#]*))?(#(.*))?)", + std::regex::ECMAScript); + + std::smatch match; + auto succeeds = std::regex_match(url, match, pathFlakeRegex); + assert(succeeds); + auto path = match[1].str(); + auto query = decodeQuery(match[3]); + auto fragment = percentDecode(match[5].str()); if (baseDir) { /* Check if 'url' is a path (either absolute or relative diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 817f77783..643d729ff 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -63,3 +63,16 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule echo '"foo"' > "$rootRepo"/submodule/sub.nix [[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] + +# Test that `nix flake metadata` parses `submodule` correctly. +cat > "$rootRepo"/flake.nix < Date: Wed, 8 Jan 2025 22:15:45 +0100 Subject: [PATCH 206/243] derivation-goal: unlock output lock to avoid deadlock guix discovered in their code base. Maybe we should do the same. --- src/libstore/build/derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2ff0ef92f..4d97250d3 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -701,6 +701,7 @@ Goal::Co DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); + outputLocks.unlock(); co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } From 83ff523865f87f61b0b2046bd3bb2d793e09911d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 12:17:09 +0100 Subject: [PATCH 207/243] parsePathFlakeRefWithFragment(): Handle query params in the non-git case Backported from lazy-trees. --- src/libflake/flake/flakeref.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 58cd45632..cdf0a40e8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -183,11 +183,13 @@ std::pair parsePathFlakeRefWithFragment( path = canonPath(path + "/" + getOr(query, "dir", "")); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); + return fromParsedURL(fetchSettings, { + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); } /** From 5f7b535b819fb2ad5f7f05bf73fee7a2e7f7c9c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 12:18:16 +0100 Subject: [PATCH 208/243] parsePathFlakeRefWithFragment(): Add unit tests --- src/libflake-tests/flakeref.cc | 50 ++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index d704a26d3..c7007a7af 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -7,18 +7,58 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ - /* ---------------------------------------------------------------------------- - * to_string - * --------------------------------------------------------------------------*/ + TEST(parseFlakeRef, path) { + fetchers::Settings fetchSettings; + + { + auto s = "/foo/bar"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + } + + { + auto s = "/foo/bar?revCount=123&rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&revCount=123"); + } + + { + auto s = "/foo/bar?xyzzy=123"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + ASSERT_EQ(fragment, "bla"); + } + + { + auto s = "/foo/bar?revCount=123#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123"); + ASSERT_EQ(fragment, "bla"); + } + } TEST(to_string, doesntReencodeUrl) { fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; auto flakeref = parseFlakeRef(fetchSettings, s); - auto parsed = flakeref.to_string(); + auto unparsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; - ASSERT_EQ(parsed, expected); + ASSERT_EQ(unparsed, expected); } } From 1a38e62a094f332776ab8b60bd3274e762f7b2cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 16:38:33 +0100 Subject: [PATCH 209/243] Remove unused variable --- src/libutil/url.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 4d4d983b8..8fb1eecfb 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -22,7 +22,6 @@ ParsedURL parseURL(const std::string & url) std::smatch match; if (std::regex_match(url, match, uriRegex)) { - auto & base = match[1]; std::string scheme = match[2]; auto authority = match[3].matched ? std::optional(match[3]) : std::nullopt; From 3ad0f45e79bc1abc96a7c05d5979552e5f936d47 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 16:42:37 +0100 Subject: [PATCH 210/243] Attempt to make the FlakeRef test succeed on macOS --- src/libflake-tests/flakeref.cc | 2 ++ src/libutil/config.hh | 1 + 2 files changed, 3 insertions(+) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index c7007a7af..2b1f5124b 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -8,6 +8,8 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ TEST(parseFlakeRef, path) { + experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); + fetchers::Settings fetchSettings; { diff --git a/src/libutil/config.hh b/src/libutil/config.hh index e98e09bf7..502d2823e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -262,6 +262,7 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } + T & get() { return value; } template bool operator ==(const U & v2) const { return value == v2; } template From fccfdbea5719f66b01870d483195f66146553b19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 20:44:25 +0100 Subject: [PATCH 211/243] nix upgrade-nix: Give a better error message if the profile is using 'nix profile' --- src/nix/upgrade-nix.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 77fb798a7..28174c4fb 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -15,7 +15,7 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { - Path profileDir; + std::filesystem::path profileDir; CmdUpgradeNix() { @@ -64,7 +64,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (profileDir == "") profileDir = getProfileDir(store); - printInfo("upgrading Nix in profile '%s'", profileDir); + printInfo("upgrading Nix in profile %s", profileDir); auto storePath = getLatestNix(store); @@ -93,7 +93,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, - fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); + fmt("installing '%s' into profile %s...", store->printStorePath(storePath), profileDir)); + + // FIXME: don't call an external process. runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -102,31 +104,33 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } /* Return the profile in which Nix is installed. */ - Path getProfileDir(ref store) + std::filesystem::path getProfileDir(ref store) { auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); const auto & where = whereOpt->parent_path(); - printInfo("found Nix in '%s'", where); + printInfo("found Nix in %s", where); if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = where.parent_path().string(); + auto profileDir = where.parent_path(); // Resolve profile to /nix/var/nix/profiles/ link. - while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) + while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) profileDir = readLink(profileDir); - printInfo("found profile '%s'", profileDir); + printInfo("found profile %s", profileDir); - Path userEnv = canonPath(profileDir, true); + Path userEnv = canonPath(profileDir.string(), true); - if (where.filename() != "bin" || - !hasSuffix(userEnv, "user-environment")) - throw Error("directory %s does not appear to be part of a Nix profile", where); + if (std::filesystem::exists(profileDir / "manifest.json")) + throw Error("directory %s is managed by 'nix profile' and currently cannot be upgraded by 'nix upgrade-nix'", profileDir); + + if (!std::filesystem::exists(profileDir / "manifest.nix")) + throw Error("directory %s does not appear to be part of a Nix profile", profileDir); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); From 96bd9bad2f434fe3489b039ac665679878460b0c Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Thu, 26 Dec 2024 20:50:36 +0800 Subject: [PATCH 212/243] fetchers/git: make path absolute for local repo --- src/libfetchers/git.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d894550c0..c4dfc27a2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -426,7 +426,16 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - repoInfo.url = repoInfo.isLocal ? url.path : url.to_string(); + // + // FIXME: here we turn a possibly relative path into an absolute path. + // This allows relative git flake inputs to be resolved against the + // **current working directory** (as in POSIX), which tends to work out + // ok in the context of flakes, but is the wrong behavior, + // as it should resolve against the flake.nix base directory instead. + // + // See: https://discourse.nixos.org/t/57783 and #9708 + // + repoInfo.url = repoInfo.isLocal ? std::filesystem::absolute(url.path).string() : url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. From 9d088fa50242c5fe2ad94bffb61a4742f232ff0b Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Fri, 27 Dec 2024 17:35:44 +0800 Subject: [PATCH 213/243] tests/flakes: check git+file:./${submodule} protocol Relative, local git repo used to work (for submodules), but it fails after 3e0129ce3b9eb094d4a3cc8023884f372f1d7ff6. This commit adds a test to prevent such failure in the future. --- tests/functional/flakes/flakes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 6c466a0c7..acbd32a91 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -106,6 +106,14 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" +# Check that relative paths are allowed for git flakes. +# This may change in the future once git submodule support is refined. +# See: https://discourse.nixos.org/t/57783 and #9708. +( + cd "$flake1Dir/.." + nix build -o "$TEST_ROOT/result" "git+file:./$(basename "$flake1Dir")" +) + # Check that store symlinks inside a flake are not interpreted as flakes. nix build -o "$flake1Dir/result" "git+file://$flake1Dir" nix path-info "$flake1Dir/result" From 37ac18d1d9faba24ce09653723342465bcac3e0b Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Wed, 1 Jan 2025 14:53:04 +0800 Subject: [PATCH 214/243] tests/flake-in-submodule: git+file:./* input --- tests/functional/flakes/flake-in-submodule.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 643d729ff..f98c19aa8 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -76,3 +76,21 @@ git -C "$rootRepo" commit -m "Add flake.nix" storePath=$(nix flake metadata --json "$rootRepo?submodules=1" | jq -r .path) [[ -e "$storePath/submodule" ]] + +# The root repo may use the submodule repo as an input +# through the relative path. This may change in the future; +# see: https://discourse.nixos.org/t/57783 and #9708. +cat > "$rootRepo"/flake.nix < Date: Fri, 10 Jan 2025 09:57:54 +0100 Subject: [PATCH 215/243] Clarify cd call in tests/functional/flakes/flakes.sh --- tests/functional/flakes/flakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index acbd32a91..995ca03b0 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -110,6 +110,7 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" # This may change in the future once git submodule support is refined. # See: https://discourse.nixos.org/t/57783 and #9708. ( + # This `cd` should not be required and is indicative of aforementioned bug. cd "$flake1Dir/.." nix build -o "$TEST_ROOT/result" "git+file:./$(basename "$flake1Dir")" ) From 22adffec347d551919eb53a3ec7a57039915346b Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Thu, 9 Jan 2025 12:50:05 -0800 Subject: [PATCH 216/243] nix flake: clarify error message when file is an unknown type --- src/libstore/nar-accessor.cc | 6 +++++- src/libutil/fs-sink.cc | 10 ++++++---- src/libutil/git.cc | 14 +++++++++++--- src/libutil/posix-source-accessor.cc | 12 ++++++++++-- src/libutil/source-accessor.cc | 20 ++++++++++++++++++++ src/libutil/source-accessor.hh | 8 ++++++-- src/nix/flake.cc | 2 +- 7 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a541bb77..c4e0b137b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -291,7 +291,11 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case SourceAccessor::Type::tMisc: + case SourceAccessor::Type::tBlock: + case SourceAccessor::Type::tChar: + case SourceAccessor::Type::tSocket: + case SourceAccessor::Type::tFifo: + case SourceAccessor::Type::tUnknown: assert(false); // cannot happen for NARs } return obj; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 72e5c731f..dd979f83f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -49,11 +49,13 @@ void copyRecursive( break; } - case SourceAccessor::tMisc: - throw Error("file '%1%' has an unsupported type", from); - + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: default: - unreachable(); + throw Error("file '%1%' has an unsupported type of %2%", from, stat.typeString()); } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index af91fa643..3303dbc32 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -200,7 +200,11 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tSymlink: return Mode::Symlink; case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; - case SourceAccessor::tMisc: return std::nullopt; + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: return std::nullopt; + case SourceAccessor::tUnknown: default: unreachable(); } } @@ -314,9 +318,13 @@ Mode dump( return Mode::Symlink; } - case SourceAccessor::tMisc: + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: default: - throw Error("file '%1%' has an unsupported type", path); + throw Error("file '%1%' has an unsupported type of %2%", path, st.typeString()); } } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 50b436893..8ee986d3f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -122,7 +122,11 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISREG(st->st_mode) ? tRegular : S_ISDIR(st->st_mode) ? tDirectory : S_ISLNK(st->st_mode) ? tSymlink : - tMisc, + S_ISCHR(st->st_mode) ? tChar : + S_ISBLK(st->st_mode) ? tBlock : + S_ISSOCK(st->st_mode) ? tSocket : + S_ISFIFO(st->st_mode) ? tFifo : + tUnknown, .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, }; @@ -156,7 +160,11 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & case std::filesystem::file_type::regular: return Type::tRegular; break; case std::filesystem::file_type::symlink: return Type::tSymlink; break; case std::filesystem::file_type::directory: return Type::tDirectory; break; - default: return tMisc; + case std::filesystem::file_type::character: return Type::tChar; break; + case std::filesystem::file_type::block: return Type::tBlock; break; + case std::filesystem::file_type::fifo: return Type::tFifo; break; + case std::filesystem::file_type::socket: return Type::tSocket; break; + default: return tUnknown; } #pragma GCC diagnostic pop }(); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d3e304f74..aa8dcf9fd 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -5,6 +5,26 @@ namespace nix { static std::atomic nextNumber{0}; +bool SourceAccessor::Stat::isNotNARSerialisable() +{ + return this->type != tRegular && this->type != tSymlink && this->type != tDirectory; +} + +std::string SourceAccessor::Stat::typeString() { + switch (this->type) { + case tRegular: return "regular"; + case tSymlink: return "symlink"; + case tDirectory: return "directory"; + case tChar: return "character device"; + case tBlock: return "block device"; + case tSocket: return "socket"; + case tFifo: return "fifo"; + case tUnknown: + default: return "unknown"; + } + return "unknown"; +} + SourceAccessor::SourceAccessor() : number(++nextNumber) , displayPrefix{"«unknown»"} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index b16960d4a..42af8256a 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -88,12 +88,13 @@ struct SourceAccessor : std::enable_shared_from_this Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. */ - tMisc + tChar, tBlock, tSocket, tFifo, + tUnknown }; struct Stat { - Type type = tMisc; + Type type = tUnknown; /** * For regular files only: the size of the file. Not all @@ -112,6 +113,9 @@ struct SourceAccessor : std::enable_shared_from_this * file in the NAR. Only returned by NAR accessors. */ std::optional narOffset; + + bool isNotNARSerialisable(); + std::string typeString(); }; Stat lstat(const CanonPath & path); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7189689cc..3da194bcc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -941,7 +941,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand createSymlink(target, os_string_to_string(PathViewNG { to2 })); } else - throw Error("file '%s' has unsupported type", from2); + throw Error("path '%s' needs to be a symlink, file, or directory but instead is a %s", from2, st.typeString()); changedFiles.push_back(to2); notice("wrote: %s", to2); } From e1613932991b719854a0b23ff0ab0b42fc9fc747 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jan 2025 16:27:40 +0100 Subject: [PATCH 217/243] Add setting 'allow-dirty-locks' This allows writing lock files with dirty inputs, so long as they have a NAR hash. (Currently they always have a NAR hash, but with lazy trees that may not always be the case.) Generally dirty locks are bad for reproducibility (we can detect if the dirty input has changed, but we have no way to fetch it except substitution). Hence we don't allow them by default. Fixes #11181. --- src/libcmd/installables.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetch-settings.hh | 16 ++++++++++++++++ src/libfetchers/fetchers.cc | 7 +++++++ src/libfetchers/fetchers.hh | 9 +++++++++ src/libflake/flake/flake.cc | 16 +++++++++++----- src/libflake/flake/flake.hh | 4 +++- src/libflake/flake/lockfile.cc | 9 +++++---- src/libflake/flake/lockfile.hh | 2 +- src/libflake/flake/settings.hh | 2 +- src/nix/flake.cc | 4 ++-- tests/functional/flakes/unlocked-override.sh | 11 ++++++++++- 12 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 250cd1413..ab3ab3104 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -450,7 +450,7 @@ ref openEvalCache( std::shared_ptr lockedFlake) { auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval - ? lockedFlake->getFingerprint(state.store) + ? lockedFlake->getFingerprint(state.store, state.fetchSettings) : std::nullopt; auto rootLoader = [&state, lockedFlake]() { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 47d1be1cc..fe42b88f1 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -182,7 +182,7 @@ static void fetchTree( if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (state.settings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isConsideredLocked(state.fetchSettings)) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f7cb34a02..2ad8aa327 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -70,6 +70,22 @@ struct Settings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + Setting allowDirtyLocks{ + this, + false, + "allow-dirty-locks", + R"( + Whether to allow dirty inputs (such as dirty Git workdirs) + to be locked via their NAR hash. This is generally bad + practice since Nix has no way to obtain such inputs if they + are subsequently modified. Therefore lock files with dirty + locks should generally only be used for local testing, and + should not be pushed to other users. + )", + {}, + true, + Xp::Flakes}; + Setting trustTarballsFromGitForges{ this, true, "trust-tarballs-from-git-forges", R"( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 87bf0dd71..35976f6a3 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -4,6 +4,7 @@ #include "fetch-to-store.hh" #include "json-utils.hh" #include "store-path-accessor.hh" +#include "fetch-settings.hh" #include @@ -154,6 +155,12 @@ bool Input::isLocked() const return scheme && scheme->isLocked(*this); } +bool Input::isConsideredLocked( + const Settings & settings) const +{ + return isLocked() || (settings.allowDirtyLocks && getNarHash()); +} + bool Input::isFinal() const { return maybeGetBoolAttr(attrs, "__final").value_or(false); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 841a44041..8cabd5103 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -95,6 +95,15 @@ public: */ bool isLocked() const; + /** + * Return whether the input is either locked, or, if + * `allow-dirty-locks` is enabled, it has a NAR hash. In the + * latter case, we can verify the input but we may not be able to + * fetch it from anywhere. + */ + bool isConsideredLocked( + const Settings & settings) const; + /** * Return whether this is a "final" input, meaning that fetching * it will not add, remove or change any attributes. (See diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 29090b900..8c0c36eb3 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -345,6 +345,7 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup } static LockFile readLockFile( + const Settings & settings, const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { @@ -380,6 +381,7 @@ LockedFlake lockFlake( } auto oldLockFile = readLockFile( + settings, state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -616,7 +618,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -678,9 +680,11 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { - if (auto unlockedInput = newLockFile.isUnlocked()) { + if (auto unlockedInput = newLockFile.isUnlocked(state.fetchSettings)) { if (lockFlags.failOnUnlocked) - throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); + throw Error( + "Will not write lock file of flake '%s' because it has an unlocked input ('%s'). " + "Use '--allow-dirty-locks' to allow this anyway.", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { @@ -979,9 +983,11 @@ static RegisterPrimOp r4({ } -std::optional LockedFlake::getFingerprint(ref store) const +std::optional LockedFlake::getFingerprint( + ref store, + const fetchers::Settings & fetchSettings) const { - if (lockFile.isUnlocked()) return std::nullopt; + if (lockFile.isUnlocked(fetchSettings)) return std::nullopt; auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 0dfd9440d..f85671a41 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -129,7 +129,9 @@ struct LockedFlake */ std::map, SourcePath> nodePaths; - std::optional getFingerprint(ref store) const; + std::optional getFingerprint( + ref store, + const fetchers::Settings & fetchSettings) const; }; struct LockFlags diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index 668ed165f..336054f5b 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -10,6 +10,7 @@ #include #include "strings.hh" +#include "flake/settings.hh" namespace nix::flake { @@ -43,8 +44,8 @@ LockedNode::LockedNode( , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input.isLocked()) - throw Error("lock file contains unlocked input '%s'", + if (!lockedRef.input.isConsideredLocked(fetchSettings)) + throw Error("Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", fetchers::attrsToJSON(lockedRef.input.toAttrs())); // For backward compatibility, lock file entries are implicitly final. @@ -228,7 +229,7 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -std::optional LockFile::isUnlocked() const +std::optional LockFile::isUnlocked(const fetchers::Settings & fetchSettings) const { std::set> nodes; @@ -247,7 +248,7 @@ std::optional LockFile::isUnlocked() const for (auto & i : nodes) { if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); - if (node && (!node->lockedRef.input.isLocked() || !node->lockedRef.input.isFinal())) + if (node && (!node->lockedRef.input.isConsideredLocked(fetchSettings) || !node->lockedRef.input.isFinal())) return node->lockedRef; } diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index a2711a516..bcc3c460c 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -71,7 +71,7 @@ struct LockFile * Check whether this lock file has any unlocked or non-final * inputs. If so, return one. */ - std::optional isUnlocked() const; + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; bool operator ==(const LockFile & other) const; diff --git a/src/libflake/flake/settings.hh b/src/libflake/flake/settings.hh index fee247a7d..991eaca1f 100644 --- a/src/libflake/flake/settings.hh +++ b/src/libflake/flake/settings.hh @@ -29,7 +29,7 @@ struct Settings : public Config this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting.", + "Whether to accept Nix configuration settings from a flake without prompting.", {}, true, Xp::Flakes}; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7189689cc..d92f644d9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -238,7 +238,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; - if (auto fingerprint = lockedFlake.getFingerprint(store)) + if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { @@ -272,7 +272,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - if (auto fingerprint = lockedFlake.getFingerprint(store)) + if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) logger->cout( ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false)); diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index ebad332d0..dcb427a8f 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -31,5 +31,14 @@ echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] +# Dirty overrides require --allow-dirty-locks. expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | - grepQuiet "cannot write lock file.*because it has an unlocked input" + grepQuiet "Will not write lock file.*because it has an unlocked input" + +nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks + +# Using a lock file with a dirty lock requires --allow-dirty-locks as well. +expectStderr 1 nix eval "$flake2Dir#x" | + grepQuiet "Lock file contains unlocked input" + +[[ $(nix eval "$flake2Dir#x" --allow-dirty-locks) = 456 ]] From 47cf93ba80b8be3701589a25ba5d8e8382b3962f Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Fri, 10 Jan 2025 18:08:27 -0800 Subject: [PATCH 218/243] Add LLVM to Flake --- flake.nix | 5 +++++ src/libcmd/package.nix | 2 +- src/libexpr/package.nix | 2 +- src/libfetchers/package.nix | 2 +- src/libflake/package.nix | 2 +- src/libmain/package.nix | 2 +- src/libstore/package.nix | 2 +- src/libutil/package.nix | 2 +- src/nix/package.nix | 2 +- 9 files changed, 13 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 83bdc5ecc..8edc2266f 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ in { inherit stdenvs native; static = native.pkgsStatic; + llvm = native.pkgsLLVM; cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); @@ -282,6 +283,7 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. @@ -321,6 +323,9 @@ prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell { pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic; })) // + prefixAttrs "llvm" (forAllStdenvs (stdenvName: makeShell { + pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsLLVM; + })) // prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell { pkgs = nixpkgsFor.${system}.cross.${crossSystem}; })) diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 53e54d2f8..9244a780a 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -76,7 +76,7 @@ mkMesonLibrary (finalAttrs: { (lib.mesonOption "readline-flavor" readlineFlavor) ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 5171d70fd..9ef56b28c 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -96,7 +96,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 7dad00025..86d505fbf 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -49,7 +49,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 92445739f..29c9fdd80 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -48,7 +48,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7d9d99b61..ede1f005b 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -45,7 +45,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 47a203f83..195efbb8a 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -87,7 +87,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 0a9f65782..11f7249ee 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -72,7 +72,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/nix/package.nix b/src/nix/package.nix index 9bc139c3b..67bf79f3f 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -99,7 +99,7 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM)) { LDFLAGS = "-fuse-ld=gold"; }; From afac093b347dbc8a745d9fbee92257bd1623426f Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Sun, 12 Jan 2025 15:01:49 +0900 Subject: [PATCH 219/243] libutil: thread-pool: ensure threads finished on error This fixes segfaults with nix copy when there was an error processing addMultipleToStore. Running with ASAN/TSAN pointed at an use-after-free with threads from the pool accessing the graph declared in processGraph after the function was exiting and destructing the variables. It turns out that if there is an error before pool.process() is called, for example while we are still enqueuing tasks, then pool.process() isn't called and threads are still left to run. By creating the pool last we ensure that it is stopped first before running other destructors even if an exception happens early. [ lix porting note: nix does not name threads so the patch has been adapted to not pass thread name ] Link: https://git.lix.systems/lix-project/lix/issues/618 Link: https://gerrit.lix.systems/c/lix/+/2355 --- src/libstore/store-api.cc | 8 ++------ src/libutil/thread-pool.hh | 5 ++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78cc3b917..3b5167730 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -246,9 +246,7 @@ void Store::addMultipleToStore( act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); }; - ThreadPool pool; - - processGraph(pool, + processGraph( storePathsToAdd, [&](const StorePath & path) { @@ -1028,12 +1026,10 @@ std::map copyPaths( } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); - ThreadPool pool; - try { // Copy the realisation closure processGraph( - pool, Realisation::closure(srcStore, toplevelRealisations), + Realisation::closure(srcStore, toplevelRealisations), [&](const Realisation & current) -> std::set { std::set children; for (const auto & [drvOutput, _] : current.dependentRealisations) { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 02765badc..dc056481a 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -83,7 +83,6 @@ private: */ template void processGraph( - ThreadPool & pool, const std::set & nodes, std::function(const T &)> getEdges, std::function processNode) @@ -97,6 +96,10 @@ void processGraph( std::function worker; + /* Create pool last to ensure threads are stopped before other destructors + * run */ + ThreadPool pool; + worker = [&](const T & node) { { From 29a1a21ce4448bc00c63d26a0c4238031c612cba Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Jan 2025 13:50:55 +0100 Subject: [PATCH 220/243] Reject merge conflicts They're usually found by other checks, but docs would remain susceptible. --- maintainers/flake-module.nix | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 6b311c62e..fcf370b71 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -10,6 +10,27 @@ # https://flake.parts/options/git-hooks-nix#options pre-commit.settings = { hooks = { + # Conflicts are usually found by other checks, but not those in docs, + # and potentially other places. + check-merge-conflicts.enable = true; + # built-in check-merge-conflicts seems ineffective against those produced by mergify backports + check-merge-conflicts-2 = { + enable = true; + entry = "${pkgs.writeScript "check-merge-conflicts" '' + #!${pkgs.runtimeShell} + conflicts=false + for file in "$@"; do + if grep --with-filename --line-number -E '^>>>>>>> ' -- "$file"; then + conflicts=true + fi + done + if $conflicts; then + echo "ERROR: found merge/patch conflicts in files" + exit 1 + fi + touch $out + ''}"; + }; clang-format = { enable = true; # https://github.com/cachix/git-hooks.nix/pull/532 From 4fac767b5295542da040d02807dc4b3a175c0337 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Sun, 12 Jan 2025 18:36:32 +0000 Subject: [PATCH 221/243] gc: replace ordered sets with unordered sets for in-memory caches During garbage collection we cache several things -- a set of known-dead paths, a set of known-alive paths, and a map of paths to their derivers. Currently they use STL maps and sets, which are ordered structures that typically are backed by binary trees. Since we are putting pseudorandom paths into these and looking them up by exact key, we don't need the ordering, and we're paying a nontrivial cost per insertion. The existing maps require O(n log n) memory and have O(log n) insertion and lookup time. We could instead use unordered maps, which are typically backed by hashmaps. These require O(n) memory and have O(1) insertion and lookup time. On my system this appears to result in a dramatic speedup -- prior to this patch I was able to delete 400k paths out of 9.5 million over the course of 34.5 hours. After this patch the same result took 89 minutes. This result should NOT be taken at face value because the two runs aren't really comparable; in particular the first started when I had 9.5 million store paths and the seconcd started with 7.8 million, so we are deleting a different set of paths starting from a much cleaner filesystem. But I do think it's indicative. Related: https://github.com/NixOS/nix/issues/9581 --- src/libstore/gc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 45dfe4ad8..ac354f3fa 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -455,7 +455,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; - StorePathSet roots, dead, alive; + std::unordered_set roots, dead, alive; struct Shared { @@ -661,7 +661,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - std::map referrersCache; + std::unordered_map referrersCache; /* Helper function that visits all paths reachable from `start` via the referrers edges and optionally derivers and derivation From fd053fdcad7bbd0dcff7a2daefd8011235653f06 Mon Sep 17 00:00:00 2001 From: Siddarth Kumar Date: Mon, 13 Jan 2025 14:12:41 +0530 Subject: [PATCH 222/243] scripts/install-multi-user: fix typo --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index b65e08682..f051ccc46 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -562,7 +562,7 @@ create_build_user_for_core() { if [ "$actual_uid" != "$uid" ]; then failure < Date: Fri, 15 Nov 2024 10:51:22 +1100 Subject: [PATCH 223/243] windows: create files if they don't exist, and with write permission --- src/libutil/fs-sink.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index dd979f83f..fadba5972 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -112,7 +112,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function Date: Tue, 14 Jan 2025 14:51:49 +0100 Subject: [PATCH 224/243] Add release note --- doc/manual/rl-next/relative-path-flakes.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/manual/rl-next/relative-path-flakes.md diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md new file mode 100644 index 000000000..5a4412316 --- /dev/null +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -0,0 +1,12 @@ +--- +synopsis: "Support for relative path inputs" +prs: [10089] +--- + +Flakes can now refer to other flakes in the same repository using relative paths, e.g. +```nix +inputs.foo.url = "path:./foo"; +``` +uses the flake in the `foo` subdirectory of the referring flake. + +This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. From ff8e2fe84e4950cd20dbc513cf6eaa1d40d6bf65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jan 2025 17:30:13 +0100 Subject: [PATCH 225/243] Fix relative 'path:' flakerefs in the CLI And handle relative 'git+file:' flakerefs while we're at it (these crashed with an assertion failure). Fixes #12248. --- src/libflake/flake/flakeref.cc | 11 ++++++++--- tests/functional/flakes/flakes.sh | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index cdf0a40e8..15d1191b3 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -102,8 +102,8 @@ std::pair parsePathFlakeRefWithFragment( if (baseDir) { /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ path = absPath(path, baseDir); @@ -232,7 +232,12 @@ std::optional> parseURLFlakeRef( ) { try { - return fromParsedURL(fetchSettings, parseURL(url), isFlake); + auto parsed = parseURL(url); + if (baseDir + && (parsed.scheme == "path" || parsed.scheme == "git+file") + && !hasPrefix(parsed.path, "/")) + parsed.path = absPath(parsed.path, *baseDir); + return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { return std::nullopt; } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 6c466a0c7..342c075a8 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -97,6 +97,9 @@ nix build -o "$TEST_ROOT/result" flake1 nix build -o "$TEST_ROOT/result" "$flake1Dir" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" ".") +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "path:.") +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "git+file:.") # Test explicit packages.default. nix build -o "$TEST_ROOT/result" "$flake1Dir#default" From ff9d886f3cb28db13bfb88d1dc4d6fe3dc83270f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jan 2025 17:42:26 +0100 Subject: [PATCH 226/243] Use isAbsolute() --- src/libexpr/eval.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/registry.cc | 2 +- src/libflake/flake/flakeref.cc | 4 ++-- src/libutil/file-system.cc | 7 +------ src/libutil/file-system.hh | 5 +++++ src/libutil/source-accessor.cc | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 21dd5a294..345c09e7e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -406,7 +406,7 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ - if (hasPrefix(uri, "/")) { + if (isAbsolute(uri)) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) rootFS2->checkAccess(CanonPath(uri)); return; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f628e042c..340e7f4fa 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme std::optional isRelative(const Input & input) const { auto path = getStrAttr(input.attrs, "path"); - if (hasPrefix(path, "/")) + if (isAbsolute(path)) return std::nullopt; else return path; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 171afcea7..c18e12d23 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -153,7 +153,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, re return std::make_shared(settings, Registry::Global); // empty registry } - if (!hasPrefix(path, "/")) { + if (!isAbsolute(path)) { auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 15d1191b3..b81eb1292 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -178,7 +178,7 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!hasPrefix(path, "/")) + if (!isAbsolute(path)) throw BadURL("flake reference '%s' is not an absolute path", url); path = canonPath(path + "/" + getOr(query, "dir", "")); } @@ -235,7 +235,7 @@ std::optional> parseURLFlakeRef( auto parsed = parseURL(url); if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file") - && !hasPrefix(parsed.path, "/")) + && !isAbsolute(parsed.path)) parsed.path = absPath(parsed.path, *baseDir); return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 923220fd0..a86f1ba4f 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -31,12 +31,7 @@ namespace nix { namespace fs { using namespace std::filesystem; } -/** - * Treat the string as possibly an absolute path, by inspecting the - * start of it. Return whether it was probably intended to be - * absolute. - */ -static bool isAbsolute(PathView path) +bool isAbsolute(PathView path) { return fs::path { path }.is_absolute(); } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 7fdaba811..204907339 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -42,6 +42,11 @@ namespace nix { struct Sink; struct Source; +/** + * Return whether the path denotes an absolute path. + */ +bool isAbsolute(PathView path); + /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index aa8dcf9fd..78f038cf3 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -115,7 +115,7 @@ CanonPath SourceAccessor::resolveSymlinks( throw Error("infinite symlink recursion in path '%s'", showPath(path)); auto target = readLink(res); res.pop(); - if (hasPrefix(target, "/")) + if (isAbsolute(target)) res = CanonPath::root; todo.splice(todo.begin(), tokenizeString>(target, "/")); } From 6a874c2865636348d192f436d22cd5a10f26a29a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 15 Jan 2025 08:59:14 -0600 Subject: [PATCH 227/243] sequoia-nixbld-user-migration: nail down PATH Fixes a user report of trouble with toybox grep and avoids potential of same basic issue with other utils. --- scripts/sequoia-nixbld-user-migration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh index 88e801706..58b5fea64 100755 --- a/scripts/sequoia-nixbld-user-migration.sh +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -2,6 +2,9 @@ set -eo pipefail +# stock path to avoid unexpected command versions +PATH="$(/usr/bin/getconf PATH)" + ((NEW_NIX_FIRST_BUILD_UID=351)) ((TEMP_NIX_FIRST_BUILD_UID=31000)) From 3d078cd508cf9fa3b623cffc2d26a6942b44b720 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 20:00:23 +0100 Subject: [PATCH 228/243] Add RossComputerGuy as `-llvm` maintainer --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 8edc2266f..9d224310b 100644 --- a/flake.nix +++ b/flake.nix @@ -283,6 +283,10 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + /** + Nix and dependencies built with LLVM and Clang, maintainers: + @RossComputerGuy, Nix team + */ "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { From cab347b4ebb2f8fe2e06a84a29c1c3027b9edc85 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 19:54:02 +0100 Subject: [PATCH 229/243] refactor: Move ld=gold rule to mesonBuildLayer --- flake.nix | 4 ---- packaging/dependencies.nix | 6 ++++++ src/libcmd/package.nix | 4 ---- src/libexpr-c/package.nix | 4 ---- src/libexpr-test-support/package.nix | 4 ---- src/libexpr-tests/package.nix | 4 ---- src/libexpr/package.nix | 2 -- src/libfetchers-tests/package.nix | 4 ---- src/libfetchers/package.nix | 4 ---- src/libflake-c/package.nix | 4 ---- src/libflake-tests/package.nix | 4 ---- src/libflake/package.nix | 4 ---- src/libmain-c/package.nix | 4 ---- src/libmain/package.nix | 4 ---- src/libstore-c/package.nix | 4 ---- src/libstore-test-support/package.nix | 4 ---- src/libstore-tests/package.nix | 4 ---- src/libstore/package.nix | 2 -- src/libutil-c/package.nix | 4 ---- src/libutil-test-support/package.nix | 4 ---- src/libutil-tests/package.nix | 4 ---- src/libutil/package.nix | 2 -- src/nix/package.nix | 4 ---- 23 files changed, 6 insertions(+), 82 deletions(-) diff --git a/flake.nix b/flake.nix index 9d224310b..8edc2266f 100644 --- a/flake.nix +++ b/flake.nix @@ -283,10 +283,6 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; - /** - Nix and dependencies built with LLVM and Clang, maintainers: - @RossComputerGuy, Nix team - */ "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 4bc7495e7..08d179b82 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -97,6 +97,12 @@ let ]; separateDebugInfo = !stdenv.hostPlatform.isStatic; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + env = prevAttrs.env or {} + // lib.optionalAttrs + (stdenv.isLinux + && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") + && !(stdenv.hostPlatform.useLLVM or false)) + { LDFLAGS = "-fuse-ld=gold"; }; }; mesonLibraryLayer = finalAttrs: prevAttrs: diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 9244a780a..5cafb4dc1 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -76,10 +76,6 @@ mkMesonLibrary (finalAttrs: { (lib.mesonOption "readline-flavor" readlineFlavor) ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 727b3a811..22726ac36 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -47,10 +47,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 4842f5f17..9726fa57a 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 70e497b7e..a4a3bb0e7 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -56,10 +56,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 9ef56b28c..3d5b78e35 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -96,8 +96,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index e18d644ed..5336672a2 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -54,10 +54,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 86d505fbf..79dd81cc9 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 7425d6140..9d339dd0a 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index f1abbb32d..51b68ad58 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -56,10 +56,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 29c9fdd80..868832b75 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -48,10 +48,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index d65792921..ba47d5cb9 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index ede1f005b..856aa660f 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -45,10 +45,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 351d4510a..e9b548473 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -47,10 +47,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 62f783c4f..705297b4c 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 4937b5329..3acf4e25c 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -62,10 +62,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = let diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 195efbb8a..4fbaea4ac 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -87,8 +87,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 4caf96804..294593b52 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -45,10 +45,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 19b5d6b77..8185a701f 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index f06704e26..28769e115 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -55,10 +55,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 11f7249ee..679872a75 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -72,8 +72,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/nix/package.nix b/src/nix/package.nix index 67bf79f3f..ac0181529 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -99,10 +99,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; From 7ad02b62e04e8845243f2c80d9891375b465d0af Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 19:58:36 +0100 Subject: [PATCH 230/243] refactor: Remove redundant parameters --- src/libexpr-c/package.nix | 1 - src/libexpr-test-support/package.nix | 1 - src/libfetchers/package.nix | 1 - src/libflake-c/package.nix | 1 - src/libflake/package.nix | 1 - src/libmain-c/package.nix | 1 - src/libmain/package.nix | 1 - src/libstore-c/package.nix | 1 - src/libstore-test-support/package.nix | 1 - src/libutil-c/package.nix | 1 - src/libutil-test-support/package.nix | 1 - src/nix/package.nix | 1 - 12 files changed, 12 deletions(-) diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 22726ac36..5047f3e2e 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-c diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 9726fa57a..48118fa0c 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-test-support diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 79dd81cc9..d4ca18555 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 9d339dd0a..dcd6c4966 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-c diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 868832b75..3fc96a20e 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index ba47d5cb9..b96901bb4 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-c diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 856aa660f..9a5b9e8c2 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , openssl diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index e9b548473..c2413c389 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-c diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 705297b4c..5d3f41b3e 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-test-support diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 294593b52..f80e0b7f0 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 8185a701f..a8a239717 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/nix/package.nix b/src/nix/package.nix index ac0181529..171621af9 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonExecutable , nix-store From 521667eb8908843925dc57e195f7702d66d0ac16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:19:20 +0100 Subject: [PATCH 231/243] Fix follow-paths test Since ff8e2fe84e4950cd20dbc513cf6eaa1d40d6bf65, 'path:' URLs on the CLI are interpreted as relative to the current directory of the user, not the path of the flake we're overriding. --- tests/functional/flakes/follow-paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 966f69cf7..a71d4c6d7 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -356,6 +356,6 @@ json=$(nix flake metadata "$flakeFollowsCustomUrlA" --json) rm "$flakeFollowsCustomUrlA"/flake.lock # if override-input is specified, lock "original" entry should contain original url -json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "path:./flakeB/flakeD" --json) +json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "$flakeFollowsCustomUrlD" --json) echo "$json" | jq .locks.nodes.C.original [[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]] From 5d03ef9caf136de06b79298b4a5f8c80a4abadbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:26:14 +0100 Subject: [PATCH 232/243] PathInputSchema::getAbsPath(): Return std::filesystem::path --- src/libfetchers/path.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9691cc9c5..351e21a1b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -91,7 +91,7 @@ struct PathInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); + writeFile(getAbsPath(input) / path.rel(), contents); } std::optional isRelative(const Input & input) const override @@ -108,12 +108,12 @@ struct PathInputScheme : InputScheme return (bool) input.getNarHash(); } - CanonPath getAbsPath(const Input & input) const + std::filesystem::path getAbsPath(const Input & input) const { auto path = getStrAttr(input.attrs, "path"); - if (path[0] == '/') - return CanonPath(path); + if (isAbsolute(path)) + return canonPath(path); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } @@ -128,7 +128,7 @@ struct PathInputScheme : InputScheme Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath.abs()); + auto storePath = store->maybeParseStorePath(absPath.string()); if (storePath) store->addTempRoot(*storePath); @@ -137,7 +137,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } @@ -159,7 +159,7 @@ struct PathInputScheme : InputScheme store object and the subpath. */ auto path = getAbsPath(input); try { - auto [storePath, subPath] = store->toStorePath(path.abs()); + auto [storePath, subPath] = store->toStorePath(path.string()); auto info = store->queryPathInfo(storePath); return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); } catch (Error &) { From 8b1fb92a0ce7fd8c0ad7955de1a59bb42866339e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:31:22 +0100 Subject: [PATCH 233/243] flakes.md: Fix indentation that broke the list --- src/nix/flake.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index e79a8ee0b..2072dd3dd 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -214,12 +214,12 @@ Currently the `type` attribute can be one of the following: hand, a flake in the `/foo` directory of a tree can use `path:../bar` to refer to the flake in `/bar`. -Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by -1. resolving them into relative paths, relative to the base directory of `flake.nix` -2. escaping URL characters (refer to IETF RFC?) -3. prepending `path:` + Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by + 1. resolving them into relative paths, relative to the base directory of `flake.nix` + 2. escaping URL characters (refer to IETF RFC?) + 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) + Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From db46d40b12cc57de47c468a73262f1c20b0374c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 13:15:20 +0100 Subject: [PATCH 234/243] Update release note --- doc/manual/rl-next/relative-path-flakes.md | 2 +- src/nix/flake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md index 5a4412316..3616f3467 100644 --- a/doc/manual/rl-next/relative-path-flakes.md +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -7,6 +7,6 @@ Flakes can now refer to other flakes in the same repository using relative paths ```nix inputs.foo.url = "path:./foo"; ``` -uses the flake in the `foo` subdirectory of the referring flake. +uses the flake in the `foo` subdirectory of the referring flake. For more information, see the documentation on [the `path` flake input type](@docroot@/command-ref/new-cli/nix3-flake.md#path-fetcher). This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. diff --git a/src/nix/flake.md b/src/nix/flake.md index 2072dd3dd..364302b61 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -187,7 +187,7 @@ Currently the `type` attribute can be one of the following: * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` * `sub/dir` (if a flake named `sub` is in the registry) -* `path`: arbitrary local directories. The required attribute `path` +* `path`: arbitrary local directories. The required attribute `path` specifies the path of the flake. The URL form is ``` From 2ca0c62a8d577e9b4bc1e6d313239e0cb9452ca2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 17:06:59 +0100 Subject: [PATCH 235/243] Remove some unnecessary quotes around std::filesystem::path --- src/libutil/file-system.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index a86f1ba4f..6fe93b63a 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -643,7 +643,7 @@ void setWriteTime( // doesn't support access time just modification time. // // System clock vs File clock issues also make that annoying. - warn("Changing file times is not yet implemented on Windows, path is '%s'", path); + warn("Changing file times is not yet implemented on Windows, path is %s", path); #elif HAVE_UTIMENSAT && HAVE_DECL_AT_SYMLINK_NOFOLLOW struct timespec times[2] = { { @@ -656,7 +656,7 @@ void setWriteTime( }, }; if (utimensat(AT_FDCWD, path.c_str(), times, AT_SYMLINK_NOFOLLOW) == -1) - throw SysError("changing modification time of '%s' (using `utimensat`)", path); + throw SysError("changing modification time of %s (using `utimensat`)", path); #else struct timeval times[2] = { { @@ -670,7 +670,7 @@ void setWriteTime( }; #if HAVE_LUTIMES if (lutimes(path.c_str(), times) == -1) - throw SysError("changing modification time of '%s'", path); + throw SysError("changing modification time of %s", path); #else bool isSymlink = optIsSymlink ? *optIsSymlink @@ -678,9 +678,9 @@ void setWriteTime( if (!isSymlink) { if (utimes(path.c_str(), times) == -1) - throw SysError("changing modification time of '%s' (not a symlink)", path); + throw SysError("changing modification time of %s (not a symlink)", path); } else { - throw Error("Cannot modification time of symlink '%s'", path); + throw Error("Cannot modification time of symlink %s", path); } #endif #endif @@ -709,7 +709,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) copyFile(entry, to / entry.path().filename(), andDelete); } } else { - throw Error("file '%s' has an unsupported type", from); + throw Error("file %s has an unsupported type", from); } setWriteTime(to, lstat(from.string().c_str())); @@ -736,7 +736,7 @@ void moveFile(const Path & oldName, const Path & newName) auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); + warn("can’t rename %s as %s, copying instead", oldName, newName); copyFile(oldPath, tempCopyTarget, true); std::filesystem::rename( os_string_to_string(PathViewNG { tempCopyTarget }), From 12e14956e22a42c26715ca084bb6b17d35628bb9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 17:22:54 +0100 Subject: [PATCH 236/243] Warn against the use of relative 'git+file:' flake inputs --- src/libfetchers/git.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c4dfc27a2..3f196da54 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -435,7 +435,16 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - repoInfo.url = repoInfo.isLocal ? std::filesystem::absolute(url.path).string() : url.to_string(); + if (repoInfo.isLocal) { + if (!isAbsolute(url.path)) { + warn( + "Fetching Git repository '%s', which uses a path relative to the current directory. " + "This is not supported and will stop working in a future release.", + url); + } + repoInfo.url = std::filesystem::absolute(url.path).string(); + } else + repoInfo.url = url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. From 3716ded8df6de1f38527b6b3fab72427f92c73f3 Mon Sep 17 00:00:00 2001 From: Andy Hamon Date: Thu, 16 Jan 2025 00:24:47 -0800 Subject: [PATCH 237/243] nix-env: add a --priority flag to --install nix-env can read priorities from a derivations meta attributes, but this only works when installing a nix expression. nix-env can also install bare store paths, however meta attributes are not readable in that case. This means that a store path can not be installed with a specific priority. Some cases where it is advantageous to install a store path: a remote host following a `nix copy`, or any time you want to save some evaluation time and happen to already know the store path. This PR addresses this shortcoming by adding a --priority flag to nix-env --install. --- .../source/command-ref/nix-env/install.md | 6 +++- src/nix-env/nix-env.cc | 35 ++++++++++++------- tests/functional/user-envs-test-case.sh | 10 +++++- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/doc/manual/source/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md index 748dd1e7a..aa5c2fbba 100644 --- a/doc/manual/source/command-ref/nix-env/install.md +++ b/doc/manual/source/command-ref/nix-env/install.md @@ -11,6 +11,7 @@ [`--from-profile` *path*] [`--preserve-installed` | `-P`] [`--remove-all` | `-r`] + [`--priority` *priority*] # Description @@ -61,6 +62,10 @@ The arguments *args* map to store paths in a number of possible ways: The derivations returned by those function calls are installed. This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. +- If `--priority` *priority* is given, the priority of the derivations being installed is set to *priority*. + This can be used to override the priority of the derivations being installed. + This is useful if *args* are [store paths], which don't have any priority information. + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. @@ -235,4 +240,3 @@ channel: ```console $ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox ``` - diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e9eb52708..fa6d6afb3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -501,9 +501,17 @@ static bool keep(PackageInfo & drv) return drv.queryMetaBool("keep", false); } +static void setMetaFlag(EvalState & state, PackageInfo & drv, + const std::string & name, const std::string & value) +{ + auto v = state.allocValue(); + v->mkString(value); + drv.setMeta(name, v); +} + static void installDerivations(Globals & globals, - const Strings & args, const Path & profile) + const Strings & args, const Path & profile, std::optional priority) { debug("installing derivations"); @@ -527,6 +535,11 @@ static void installDerivations(Globals & globals, newNames.insert(DrvName(i.queryName()).name); } + if (priority) { + for (auto & drv : newElems) { + setMetaFlag(*globals.state, drv, "priority", std::to_string((priority.value()))); + } + } while (true) { auto lockToken = optimisticLockProfile(profile); @@ -564,6 +577,7 @@ static void installDerivations(Globals & globals, static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) { + std::optional priority; for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { auto arg = *i++; if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; @@ -571,10 +585,15 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) globals.preserveInstalled = true; else if (arg == "--remove-all" || arg == "-r") globals.removeAll = true; + else if (arg == "--priority") { + if (i == opFlags.end()) + throw UsageError("'%1%' requires an argument", arg); + priority = std::stoi(*i++); + } else throw UsageError("unknown flag '%1%'", arg); } - installDerivations(globals, opArgs, globals.profile); + installDerivations(globals, opArgs, globals.profile, priority); } @@ -689,15 +708,6 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) } -static void setMetaFlag(EvalState & state, PackageInfo & drv, - const std::string & name, const std::string & value) -{ - auto v = state.allocValue(); - v->mkString(value); - drv.setMeta(name, v); -} - - static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) @@ -1507,7 +1517,8 @@ static int main_nix_env(int argc, char * * argv) opFlags.push_back(*arg); /* FIXME: hacky */ if (*arg == "--from-profile" || - (op == opQuery && (*arg == "--attr" || *arg == "-A"))) + (op == opQuery && (*arg == "--attr" || *arg == "-A")) || + (op == opInstall && (*arg == "--priority"))) opFlags.push_back(getArg(*arg, arg, end)); } else diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index 117c6c7a4..3483a4600 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -173,13 +173,21 @@ nix-env -q '*' | grepQuiet bar-0.1.1 # Test priorities: foo-0.1 has a lower priority than foo-1.0, so it # should be possible to install both without a collision. Also test -# ‘--set-flag priority’ to manually override the declared priorities. +# '-i --priority' and '--set-flag priority' to manually override the +# declared priorities. nix-env -e '*' nix-env -i foo-0.1 foo-1.0 [ "$($profiles/test/bin/foo)" = "foo-1.0" ] nix-env --set-flag priority 1 foo-0.1 [ "$($profiles/test/bin/foo)" = "foo-0.1" ] +# Priorities can be overridden with the --priority flag +nix-env -e '*' +nix-env -i foo-1.0 +[ "$($profiles/test/bin/foo)" = "foo-1.0" ] +nix-env -i --priority 1 foo-0.1 +[ "$($profiles/test/bin/foo)" = "foo-0.1" ] + # Test nix-env --set. nix-env --set $outPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] From 6ea339ce8a5f0e925f601990c9fb0c700be29187 Mon Sep 17 00:00:00 2001 From: Andy Hamon Date: Thu, 16 Jan 2025 11:53:43 -0800 Subject: [PATCH 238/243] Update src/nix-env/nix-env.cc Co-authored-by: Eelco Dolstra --- src/nix-env/nix-env.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index fa6d6afb3..c99c1088e 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -588,7 +588,9 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--priority") { if (i == opFlags.end()) throw UsageError("'%1%' requires an argument", arg); - priority = std::stoi(*i++); + priority = string2Int(*i++); + if (!priority) + throw UsageError("'--priority' requires an integer argument"); } else throw UsageError("unknown flag '%1%'", arg); } From 5807b5cda0198ec96688039eab42ec0e6a86ed27 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 23:44:18 +0100 Subject: [PATCH 239/243] Trivial changes from lazy-trees Rename allowLookup -> useRegistries, rename lockRootPath -> followsPrefix, drop an unnecessary getFlake() variant. --- src/libflake/flake/flake.cc | 53 ++++++++++++++++++++----------------- src/libflake/flake/flake.hh | 2 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 4e70b4089..cac39f8ab 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -43,7 +43,7 @@ static std::optional lookupInFlakeCache( static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, - bool allowLookup, + bool useRegistries, FlakeCache & flakeCache) { auto fetched = lookupInFlakeCache(flakeCache, originalRef); @@ -54,7 +54,7 @@ static std::tuple fetchOrSubstituteTree( auto [storePath, lockedRef] = originalRef.fetchTree(state.store); fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { - if (allowLookup) { + if (useRegistries) { resolvedRef = originalRef.resolve( state.store, [](fetchers::Registry::RegistryType type) { @@ -105,7 +105,7 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir); static FlakeInput parseFlakeInput( @@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput( std::string_view inputName, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir) { expectType(state, nAttrs, *value, pos); @@ -220,7 +220,7 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir) { std::map inputs; @@ -345,25 +345,20 @@ static Flake readFlake( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - bool allowLookup, + bool useRegistries, FlakeCache & flakeCache, - InputPath lockRootPath) + const InputPath & lockRootPath) { auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, allowLookup, flakeCache); + state, originalRef, useRegistries, flakeCache); return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) -{ - return getFlake(state, originalRef, allowLookup, flakeCache, {}); -} - -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries) { FlakeCache flakeCache; - return getFlake(state, originalRef, allowLookup, flakeCache); + return getFlake(state, originalRef, useRegistries, flakeCache, {}); } static LockFile readLockFile( @@ -390,7 +385,7 @@ LockedFlake lockFlake( auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); - auto flake = getFlake(state, topRef, useRegistries, flakeCache); + auto flake = getFlake(state, topRef, useRegistries, flakeCache, {}); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -445,7 +440,7 @@ LockedFlake lockFlake( ref node, const InputPath & inputPathPrefix, std::shared_ptr oldNode, - const InputPath & lockRootPath, + const InputPath & followsPrefix, const SourcePath & sourcePath, bool trustLock)> computeLocks; @@ -461,7 +456,11 @@ LockedFlake lockFlake( /* The old node, if any, from which locks can be copied. */ std::shared_ptr oldNode, - const InputPath & lockRootPath, + /* The prefix relative to which 'follows' should be + interpreted. When a node is initially locked, it's + relative to the node's flake; when it's already locked, + it's relative to the root of the lock file. */ + const InputPath & followsPrefix, /* The source path of this node's flake. */ const SourcePath & sourcePath, bool trustLock) @@ -633,7 +632,7 @@ LockedFlake lockFlake( break; } } - auto absoluteFollows(lockRootPath); + auto absoluteFollows(followsPrefix); absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); fakeInputs.emplace(i.first, FlakeInput { .follows = absoluteFollows, @@ -645,9 +644,10 @@ LockedFlake lockFlake( if (mustRefetch) { auto inputFlake = getInputFlake(); nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, followsPrefix, + inputFlake.path, false); } else { - computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); + computeLocks(fakeInputs, childNode, inputPath, oldLock, followsPrefix, sourcePath, true); } } else { @@ -672,7 +672,11 @@ LockedFlake lockFlake( if (input.isFlake) { auto inputFlake = getInputFlake(); - auto childNode = make_ref(inputFlake.lockedRef, ref, true, overridenParentPath); + auto childNode = make_ref( + inputFlake.lockedRef, + ref, + true, + overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -693,7 +697,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), - oldLock ? lockRootPath : inputPath, + oldLock ? followsPrefix : inputPath, inputFlake.path, false); } @@ -817,8 +821,7 @@ LockedFlake lockFlake( repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; - FlakeCache dummyCache; - flake = getFlake(state, topRef, useRegistries, dummyCache); + flake = getFlake(state, topRef, useRegistries); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index f85671a41..9ab661fce 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -110,7 +110,7 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool useRegistries); /** * Fingerprint of a locked flake; used as a cache key. From f0271090b24035ff5f19899f9971519854738e3b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 23:47:57 +0100 Subject: [PATCH 240/243] Drop unused 'settings' argument --- src/libflake/flake/flake.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index cac39f8ab..873a26d5c 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -362,7 +362,6 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistri } static LockFile readLockFile( - const Settings & settings, const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { @@ -398,7 +397,6 @@ LockedFlake lockFlake( } auto oldLockFile = readLockFile( - settings, state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -696,7 +694,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? followsPrefix : inputPath, inputFlake.path, false); From 3197c19a31f5ce42b099d257b6ba22fb25e707c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 12:33:23 +0100 Subject: [PATCH 241/243] Add link to tracking issue --- src/libfetchers/git.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3f196da54..5a9679199 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -439,7 +439,8 @@ struct GitInputScheme : InputScheme if (!isAbsolute(url.path)) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " - "This is not supported and will stop working in a future release.", + "This is not supported and will stop working in a future release. " + "See https://github.com/NixOS/nix/issues/12281 for details.", url); } repoInfo.url = std::filesystem::absolute(url.path).string(); From 9003343b5345622399571d8e232b7f94ec3aec28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 13:11:07 +0100 Subject: [PATCH 242/243] Git fetcher: Replace RepoInfo::url by a std::variant Previously the 'url' field was either a path or a URL, depending on 'isLocal'. This replaces both fields by a std::variant, which is more type-safe. --- src/libfetchers/git.cc | 122 +++++++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5a9679199..bca4b2d38 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -297,7 +297,7 @@ struct GitInputScheme : InputScheme Strings args = {"clone"}; - args.push_back(repoInfo.url); + args.push_back(repoInfo.locationToArg()); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -311,11 +311,14 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } + // FIXME: return std::filesystem::path std::optional getSourcePath(const Input & input) const override { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal) return repoInfo.url; - return std::nullopt; + if (auto path = repoInfo.getPath()) + return *path; + else + return std::nullopt; } void putFile( @@ -325,14 +328,15 @@ struct GitInputScheme : InputScheme std::optional commitMsg) const override { auto repoInfo = getRepoInfo(input); - if (!repoInfo.isLocal) + auto repoPath = repoInfo.getPath(); + if (!repoPath) throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - writeFile((CanonPath(repoInfo.url) / path).abs(), contents); + writeFile(*repoPath / path.rel(), contents); auto result = runProgram(RunOptions { .program = "git", - .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + .args = {"-C", *repoPath, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, }); auto exitCode = #ifndef WIN32 // TODO abstract over exit status handling on Windows @@ -345,7 +349,7 @@ struct GitInputScheme : InputScheme if (exitCode != 0) { // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) { @@ -353,7 +357,7 @@ struct GitInputScheme : InputScheme logger->pause(); Finally restoreLogger([]() { logger->resume(); }); runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, + { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, *commitMsg); } } @@ -361,24 +365,41 @@ struct GitInputScheme : InputScheme struct RepoInfo { - /* Whether this is a local, non-bare repository. */ - bool isLocal = false; + /* Either the path of the repo (for local, non-bare repos), or + the URL (which is never a `file` URL). */ + std::variant location; /* Working directory info: the complete list of files, and whether the working directory is dirty compared to HEAD. */ GitRepo::WorkdirInfo workdirInfo; - /* URL of the repo, or its path if isLocal. Never a `file` URL. */ - std::string url; + std::string locationToArg() const + { + return std::visit( + overloaded { + [&](const std::filesystem::path & path) + { return path.string(); }, + [&](const ParsedURL & url) + { return url.to_string(); } + }, location); + } + + std::optional getPath() const + { + if (auto path = std::get_if(&location)) + return *path; + else + return std::nullopt; + } void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", url); + throw Error("Git tree '%s' is dirty", locationToArg()); if (settings.warnDirty) - warn("Git tree '%s' is dirty", url); + warn("Git tree '%s' is dirty", locationToArg()); } } @@ -425,7 +446,6 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; // // FIXME: here we turn a possibly relative path into an absolute path. // This allows relative git flake inputs to be resolved against the @@ -435,7 +455,7 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - if (repoInfo.isLocal) { + if (url.scheme == "file" && !forceHttp && !isBareRepository) { if (!isAbsolute(url.path)) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " @@ -443,14 +463,14 @@ struct GitInputScheme : InputScheme "See https://github.com/NixOS/nix/issues/12281 for details.", url); } - repoInfo.url = std::filesystem::absolute(url.path).string(); + repoInfo.location = std::filesystem::absolute(url.path); } else - repoInfo.url = url.to_string(); + repoInfo.location = url; // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); + if (auto repoPath = repoInfo.getPath(); !input.getRef() && !input.getRev() && repoPath) + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(*repoPath); return repoInfo; } @@ -480,7 +500,7 @@ struct GitInputScheme : InputScheme if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); - Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg())); auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); @@ -491,11 +511,15 @@ struct GitInputScheme : InputScheme std::string getDefaultRef(const RepoInfo & repoInfo) const { - auto head = repoInfo.isLocal - ? GitRepo::openRepo(repoInfo.url)->getWorkdirRef() - : readHeadCached(repoInfo.url); + auto head = std::visit( + overloaded { + [&](const std::filesystem::path & path) + { return GitRepo::openRepo(path)->getWorkdirRef(); }, + [&](const ParsedURL & url) + { return readHeadCached(url.to_string()); } + }, repoInfo.location); if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg()); return "master"; } return *head; @@ -540,12 +564,13 @@ struct GitInputScheme : InputScheme Path repoDir; - if (repoInfo.isLocal) { - repoDir = repoInfo.url; + if (auto repoPath = repoInfo.getPath()) { + repoDir = *repoPath; if (!input.getRev()) input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { - Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); + auto repoUrl = std::get(repoInfo.location); + Path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input)); repoDir = cacheDir; repoInfo.gitDir = "."; @@ -555,7 +580,7 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(cacheDir, true, true); // We need to set the origin so resolving submodule URLs works - repo->setRemote("origin", repoInfo.url); + repo->setRemote("origin", repoUrl.to_string()); Path localRefFile = ref.compare(0, 5, "refs/") == 0 @@ -594,11 +619,11 @@ struct GitInputScheme : InputScheme ? ref : "refs/heads/" + ref; - repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); + repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; logError(e.info()); - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.locationToArg()); } try { @@ -607,8 +632,8 @@ struct GitInputScheme : InputScheme } catch (Error & e) { warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg); } - if (!originalRef && !storeCachedHead(repoInfo.url, ref)) - warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); + if (!originalRef && !storeCachedHead(repoUrl.to_string(), ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); } if (auto rev = input.getRev()) { @@ -620,8 +645,7 @@ struct GitInputScheme : InputScheme "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", rev->gitRev(), ref, - repoInfo.url - ); + repoInfo.locationToArg()); } else input.attrs.insert_or_assign("rev", repo->resolveRef(ref).gitRev()); @@ -633,7 +657,7 @@ struct GitInputScheme : InputScheme auto isShallow = repo->isShallow(); if (isShallow && !getShallowAttr(input)) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.locationToArg()); // FIXME: check whether rev is an ancestor of ref? @@ -648,7 +672,7 @@ struct GitInputScheme : InputScheme infoAttrs.insert_or_assign("revCount", getRevCount(repoInfo, repoDir, rev)); - printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); verifyCommit(input, repo); @@ -702,21 +726,23 @@ struct GitInputScheme : InputScheme RepoInfo & repoInfo, Input && input) const { + auto repoPath = repoInfo.getPath().value(); + if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) repoInfo.workdirInfo.files.insert(submodule.path); - auto repo = GitRepo::openRepo(repoInfo.url, false, false); + auto repo = GitRepo::openRepo(repoPath, false, false); auto exportIgnore = getExportIgnoreAttr(input); ref accessor = repo->getAccessor(repoInfo.workdirInfo, exportIgnore, - makeNotAllowedError(repoInfo.url)); + makeNotAllowedError(repoInfo.locationToArg())); - accessor->setPathDisplay(repoInfo.url); + accessor->setPathDisplay(repoInfo.locationToArg()); /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the @@ -725,10 +751,10 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { - auto submodulePath = CanonPath(repoInfo.url) / submodule.path; + auto submodulePath = repoPath / submodule.path.rel(); fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", submodulePath.abs()); + attrs.insert_or_assign("url", submodulePath.string()); attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out @@ -752,7 +778,7 @@ struct GitInputScheme : InputScheme } if (!repoInfo.workdirInfo.isDirty) { - auto repo = GitRepo::openRepo(repoInfo.url); + auto repo = GitRepo::openRepo(repoPath); if (auto ref = repo->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); @@ -762,7 +788,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("revCount", - rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev)); + rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); verifyCommit(input, repo); } else { @@ -781,7 +807,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign( "lastModified", repoInfo.workdirInfo.headRev - ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) : 0); return {accessor, std::move(input)}; @@ -804,7 +830,7 @@ struct GitInputScheme : InputScheme } auto [accessor, final] = - input.getRef() || input.getRev() || !repoInfo.isLocal + input.getRef() || input.getRev() || !repoInfo.getPath() ? getAccessorFromCommit(store, repoInfo, std::move(input)) : getAccessorFromWorkdir(store, repoInfo, std::move(input)); @@ -822,14 +848,14 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); From f5548c17ed61bf6e6392ece9c83eb76c9f31cf66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 13:17:58 +0100 Subject: [PATCH 243/243] getSourcePath(): Return std::filesystem::path --- src/libfetchers/fetchers.cc | 4 ++-- src/libfetchers/fetchers.hh | 4 ++-- src/libfetchers/git.cc | 9 ++------- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 4 ++-- src/libflake/flake/flake.cc | 2 +- src/nix/develop.cc | 2 +- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 1cfc48d52..9459db087 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -358,7 +358,7 @@ void Input::clone(const Path & destDir) const scheme->clone(*this, destDir); } -std::optional Input::getSourcePath() const +std::optional Input::getSourcePath() const { assert(scheme); return scheme->getSourcePath(*this); @@ -461,7 +461,7 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) const +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 87c7b08c0..644c267c1 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -164,7 +164,7 @@ public: void clone(const Path & destDir) const; - std::optional getSourcePath() const; + std::optional getSourcePath() const; /** * Write a file to this input, for input types that support @@ -247,7 +247,7 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input) const; + virtual std::optional getSourcePath(const Input & input) const; virtual void putFile( const Input & input, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index bca4b2d38..4523f49ca 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -311,14 +311,9 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - // FIXME: return std::filesystem::path - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - auto repoInfo = getRepoInfo(input); - if (auto path = repoInfo.getPath()) - return *path; - else - return std::nullopt; + return getRepoInfo(input).getPath(); } void putFile( diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index c2fd8139c..61cbca202 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -126,7 +126,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 351e21a1b..9d1cce0f3 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -80,9 +80,9 @@ struct PathInputScheme : InputScheme }; } - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - return getStrAttr(input.attrs, "path"); + return getAbsPath(input); } void putFile( diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 873a26d5c..c2145ab39 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -781,7 +781,7 @@ LockedFlake lockFlake( writeFile(*lockFlags.outputLockFilePath, newLockFileS); } else { auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto outputLockFilePath = *sourcePath + "/" + relPath; + auto outputLockFilePath = *sourcePath / relPath; bool lockFileExists = pathExists(outputLockFilePath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1736add9a..deee89aa1 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -696,7 +696,7 @@ struct CmdDevelop : Common, MixEnvironment auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { if (chdir(sourcePath->c_str()) == -1) { - throw SysError("chdir to '%s' failed", *sourcePath); + throw SysError("chdir to %s failed", *sourcePath); } } }