nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-23 18:59:35 +01:00
Code Activity

Merge commit '734019ce56' into progress-bar

Browse source
This commit is contained in:
John Ericson 2023-03-11 17:11:20 -05:00
commit 37e74bb69b
92 changed files with 1593 additions and 687 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -90,6 +90,7 @@ perl/Makefile.config
/misc/systemd/nix-daemon.service
/misc/systemd/nix-daemon.socket
/misc/systemd/nix-daemon.conf
/misc/upstart/nix-daemon.conf
/src/resolve-system-dependencies/resolve-system-dependencies

3
doc/manual/generate-options.nix
View file

@ -6,7 +6,8 @@ options:
concatStrings (map
(name:
let option = options.${name}; in
" - `${name}` \n\n"
" - [`${name}`](#conf-${name})"
+ "<p id=\"conf-${name}\"></p>\n\n"
+ concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n"
+ (if option.documentDefault
then " **Default:** " + (

92
doc/manual/src/installation/installing-binary.md
View file

@ -84,7 +84,9 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
The installer will first back up these files with a `.backup-before-nix`
extension. The installer will also create `/etc/profile.d/nix.sh`.
You can uninstall Nix with the following commands:
## Uninstalling
### Linux
```console
sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
@ -95,15 +97,95 @@ sudo systemctl stop nix-daemon.service
sudo systemctl disable nix-daemon.socket
sudo systemctl disable nix-daemon.service
sudo systemctl daemon-reload
# If you are on macOS, you will need to run:
sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
```
There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
and `/etc/zshrc` which you may remove.
### macOS
1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing
`nix-daemon.sh`, which should look like this:
```bash
# Nix
if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then
. '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh'
fi
# End Nix
```
If these files haven't been altered since installing Nix you can simply put
the backups back in place:
```console
sudo mv /etc/zshrc.backup-before-nix /etc/zshrc
sudo mv /etc/bashrc.backup-before-nix /etc/bashrc
```
This will stop shells from sourcing the file and bringing everything you
installed using Nix in scope.
2. Stop and remove the Nix daemon services:
```console
sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist
sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist
```
This stops the Nix daemon and prevents it from being started next time you
boot the system.
3. Remove the `nixbld` group and the `_nixbuildN` users:
```console
sudo dscl . -delete /Groups/nixbld
for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done
```
This will remove all the build users that no longer serve a purpose.
4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store
volume on `/nix`, which looks like this,
`LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic
mounting of the Nix Store volume.
5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only
line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`.
This will prevent the creation of the empty `/nix` directory to provide a
mountpoint for the Nix Store volume.
6. Remove the files Nix added to your system:
```console
sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
```
This gets rid of any data Nix may have created except for the store which is
removed next.
7. Remove the Nix Store volume:
```console
sudo diskutil apfs deleteVolume /nix
```
This will remove the Nix Store volume and everything that was added to the
store.
> **Note**
>
> After you complete the steps here, you will still have an empty `/nix`
> directory. This is an expected sign of a successful uninstall. The empty
> `/nix` directory will disappear the next time you reboot.
>
> You do not have to reboot to finish uninstalling Nix. The uninstall is
> complete. macOS (Catalina+) directly controls root directories and its
> read-only root will prevent you from manually deleting the empty `/nix`
> mountpoint.
# macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a>
<!-- Note: anchors above to catch permalinks to old explanations -->

2
doc/manual/src/release-notes/rl-2.7.md
View file

@ -1,4 +1,4 @@
# Release X.Y (2022-03-07)
# Release 2.7 (2022-03-07)
* Nix will now make some helpful suggestions when you mistype
something on the command line. For instance, if you type `nix build

39
doc/manual/src/release-notes/rl-next.md
View file

@ -1,3 +1,42 @@
# Release X.Y (202?-??-??)
* Various nix commands can now read expressions from stdin with `--file -`.
* `nix store make-content-addressable` has been renamed to `nix store
make-content-addressed`.
* New experimental builtin function `builtins.fetchClosure` that
copies a closure from a binary cache at evaluation time and rewrites
it to content-addressed form (if it isn't already). Like
`builtins.storePath`, this allows importing pre-built store paths;
the difference is that it doesn't require the user to configure
binary caches and trusted public keys.
This function is only available if you enable the experimental
feature `fetch-closure`.
* New experimental feature: *impure derivations*. These are
derivations that can produce a different result every time they're
built. Here is an example:
```nix
stdenv.mkDerivation {
name = "impure";
__impure = true; # marks this derivation as impure
buildCommand = "date > $out";
}
```
Running `nix build` twice on this expression will build the
derivation twice, producing two different content-addressed store
paths. Like fixed-output derivations, impure derivations have access
to the network. Only fixed-output derivations and impure derivations
can depend on an impure derivation.
* The `nixosModule` flake output attribute has been renamed consistent
with the `.default` renames in nix 2.7.
* `nixosModule` → `nixosModules.default`
As before, the old output will continue to work, but `nix flake check` will
issue a warning about it.

6
flake.lock generated
View file

@ -18,11 +18,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1632864508,
"narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
"lastModified": 1645296114,
"narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
"rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
"type": "github"
},
"original": {

3
misc/systemd/local.mk
View file

@ -1,7 +1,8 @@
ifdef HOST_LINUX
$(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
$(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf
endif

1
misc/systemd/nix-daemon.conf.in Normal file
View file

@ -0,0 +1 @@
d @localstatedir@/nix/daemon-socket 0755 root root - -

1
misc/systemd/nix-daemon.service.in
View file

@ -3,6 +3,7 @@ Description=Nix Daemon
Documentation=man:nix-daemon https://nixos.org/manual
RequiresMountsFor=@storedir@
RequiresMountsFor=@localstatedir@
RequiresMountsFor=@localstatedir@/nix/db
ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
[Service]

2
scripts/check-hydra-status.sh
View file

@ -14,7 +14,7 @@ curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master
someBuildFailed=0
for buildId in $BUILDS_FOR_LATEST_EVAL; do
buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
finished=$(echo "$buildInfo" | jq -r '.finished')

2
scripts/install-multi-user.sh
View file

@ -739,7 +739,7 @@ install_from_extracted_nix() {
cd "$EXTRACTED_NIX_PATH"
_sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
cp -RLp ./store/* "$NIX_ROOT/store/"
cp -RPp ./store/* "$NIX_ROOT/store/"
_sudo "to make the new store non-writable at $NIX_ROOT/store" \
chmod -R ugo-w "$NIX_ROOT/store/"

9
scripts/install-systemd-multi-user.sh
View file

@ -9,6 +9,8 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
readonly TMPFILES_SRC=/lib/tmpfiles.d/nix-daemon.conf
readonly TMPFILES_DEST=/etc/tmpfiles.d/nix-daemon.conf
# Path for the systemd override unit file to contain the proxy settings
readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
@ -83,6 +85,13 @@ EOF
poly_configure_nix_daemon_service() {
if [ -e /run/systemd/system ]; then
task "Setting up the nix-daemon systemd service"
_sudo "to create the nix-daemon tmpfiles config" \
ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
_sudo "to run systemd-tmpfiles once to pick that path up" \
systemd-tmpfiles --create --prefix=/nix/var/nix
_sudo "to set up the nix-daemon service" \
systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"

2
scripts/install.in
View file

@ -82,7 +82,7 @@ if [ "$(uname -s)" != "Darwin" ]; then
fi
if command -v curl > /dev/null 2>&1; then
fetch() { curl -L "$1" -o "$2"; }
fetch() { curl --fail -L "$1" -o "$2"; }
elif command -v wget > /dev/null 2>&1; then
fetch() { wget "$1" -O "$2"; }
else

3
src/libcmd/command.cc
View file

@ -204,7 +204,8 @@ Strings editorFor(const Pos & pos)
if (pos.line > 0 && (
editor.find("emacs") != std::string::npos ||
editor.find("nano") != std::string::npos ||
editor.find("vim") != std::string::npos))
editor.find("vim") != std::string::npos ||
editor.find("kak") != std::string::npos))
args.push_back(fmt("+%d", pos.line));
args.push_back(pos.file);
return args;

139
src/libcmd/installables.cc
View file

@ -758,55 +758,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
return installables.front();
}
BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
BuiltPaths Installable::build(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode)
{
BuiltPaths res;
for (const auto & b : hopefullyBuiltPaths)
std::visit(
overloaded{
[&](const DerivedPath::Opaque & bo) {
res.push_back(BuiltPath::Opaque{bo.path});
},
[&](const DerivedPath::Built & bfd) {
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*store);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto outputId =
DrvOutput{outputHashes.at(output), output};
auto realisation =
store->queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
outputs.insert_or_assign(
output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
outputs.insert_or_assign(
output, *drvOutputs.at(output).second);
}
}
res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
},
},
b.raw());
for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
res.push_back(builtPath);
return res;
}
BuiltPaths Installable::build(
std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
@ -817,39 +782,93 @@ BuiltPaths Installable::build(
settings.readOnlyMode = true;
std::vector<DerivedPath> pathsToBuild;
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
for (auto & i : installables) {
auto b = i->toDerivedPaths();
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
for (auto b : i->toDerivedPaths()) {
pathsToBuild.push_back(b);
backmap[b].push_back(i);
}
}
std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
switch (mode) {
case Realise::Nothing:
case Realise::Derivation:
printMissing(store, pathsToBuild, lvlError);
return getBuiltPaths(evalStore, store, pathsToBuild);
for (auto & path : pathsToBuild) {
for (auto & installable : backmap[path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*store);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
DrvOutput outputId { outputHashes.at(output), output };
auto realisation = store->queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
outputs.insert_or_assign(
output, *drvOutputs.at(output).second);
}
}
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
},
[&](const DerivedPath::Opaque & bo) {
res.push_back({installable, BuiltPath::Opaque { bo.path }});
},
}, path.raw());
}
}
break;
case Realise::Outputs: {
BuiltPaths res;
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success())
buildResult.rethrow();
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back(BuiltPath::Built { bfd.drvPath, outputs });
},
[&](const DerivedPath::Opaque & bo) {
res.push_back(BuiltPath::Opaque { bo.path });
},
}, buildResult.path.raw());
for (auto & installable : backmap[buildResult.path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
std::map<std::string, StorePath> outputs;
for (auto & path : buildResult.builtOutputs)
outputs.emplace(path.first.outputName, path.second.outPath);
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
},
[&](const DerivedPath::Opaque & bo) {
res.push_back({installable, BuiltPath::Opaque { bo.path }});
},
}, buildResult.path.raw());
}
}
return res;
break;
}
default:
assert(false);
}
return res;
}
BuiltPaths Installable::toBuiltPaths(

12
src/libcmd/installables.hh
View file

@ -98,6 +98,13 @@ struct Installable
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode = bmNormal);
static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode = bmNormal);
static std::set<StorePath> toStorePaths(
ref<Store> evalStore,
ref<Store> store,
@ -185,9 +192,4 @@ ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake);
BuiltPaths getBuiltPaths(
ref<Store> evalStore,
ref<Store> store,
const DerivedPaths & hopefullyBuiltPaths);
}

21
src/libexpr/eval-cache.cc
View file

@ -21,6 +21,8 @@ struct AttrDb
{
std::atomic_bool failed{false};
const Store & cfg;
struct State
{
SQLite db;
@ -33,8 +35,9 @@ struct AttrDb
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
AttrDb(const Store & cfg, const Hash & fingerprint)
: cfg(cfg)
, _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
@ -254,10 +257,10 @@ struct AttrDb
return {{rowId, attrs}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
NixStringContext context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
context.push_back(decodeContext(cfg, s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
@ -274,10 +277,10 @@ struct AttrDb
}
};
static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint)
{
try {
return std::make_shared<AttrDb>(fingerprint);
return std::make_shared<AttrDb>(cfg, fingerprint);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
@ -288,7 +291,7 @@ EvalCache::EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader)
: db(useCache ? makeAttrDb(*useCache) : nullptr)
: db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
@ -546,7 +549,7 @@ string_t AttrCursor::getStringWithContext()
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true;
for (auto & c : s->second) {
if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
if (!root->state.store->isValidPath(c.first)) {
valid = false;
break;
}
@ -563,7 +566,7 @@ string_t AttrCursor::getStringWithContext()
auto & v = forceValue();
if (v.type() == nString)
return {v.string.s, v.getContext()};
return {v.string.s, v.getContext(*root->state.store)};
else if (v.type() == nPath)
return {v.path, {}};
else

2
src/libexpr/eval-cache.hh
View file

@ -52,7 +52,7 @@ struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::pair<std::string, NixStringContext> string_t;
typedef std::variant<
std::vector<Symbol>,

81
src/libexpr/eval.cc
View file

@ -96,20 +96,20 @@ RootValue allocRootValue(Value * v)
}
void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v)
void Value::print(std::ostream & str, std::set<const void *> * seen) const
{
checkInterrupt();
switch (v.internalType) {
switch (internalType) {
case tInt:
str << v.integer;
str << integer;
break;
case tBool:
str << (v.boolean ? "true" : "false");
str << (boolean ? "true" : "false");
break;
case tString:
str << "\"";
for (const char * i = v.string.s; *i; i++)
for (const char * i = string.s; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
@ -119,19 +119,19 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
str << "\"";
break;
case tPath:
str << v.path; // !!! escaping?
str << path; // !!! escaping?
break;
case tNull:
str << "null";
break;
case tAttrs: {
if (!v.attrs->empty() && !seen.insert(v.attrs).second)
str << "<REPEAT>";
if (seen && !attrs->empty() && !seen->insert(attrs).second)
str << "«repeated»";
else {
str << "{ ";
for (auto & i : v.attrs->lexicographicOrder()) {
for (auto & i : attrs->lexicographicOrder()) {
str << i->name << " = ";
printValue(str, seen, *i->value);
i->value->print(str, seen);
str << "; ";
}
str << "}";
@ -141,12 +141,12 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
case tList1:
case tList2:
case tListN:
if (v.listSize() && !seen.insert(v.listElems()).second)
str << "<REPEAT>";
if (seen && listSize() && !seen->insert(listElems()).second)
str << "«repeated»";
else {
str << "[ ";
for (auto v2 : v.listItems()) {
printValue(str, seen, *v2);
for (auto v2 : listItems()) {
v2->print(str, seen);
str << " ";
}
str << "]";
@ -166,10 +166,10 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
str << "<PRIMOP-APP>";
break;
case tExternal:
str << *v.external;
str << *external;
break;
case tFloat:
str << v.fpoint;
str << fpoint;
break;
default:
abort();
@ -177,10 +177,16 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
}
std::ostream & operator << (std::ostream & str, const Value & v)
void Value::print(std::ostream & str, bool showRepeated) const
{
std::set<const void *> seen;
printValue(str, seen, v);
print(str, showRepeated ? nullptr : &seen);
}
std::ostream & operator << (std::ostream & str, const Value & v)
{
v.print(str, false);
return str;
}
@ -430,6 +436,7 @@ EvalState::EvalState(
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
, sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
@ -501,23 +508,6 @@ EvalState::~EvalState()
}
void EvalState::requireExperimentalFeatureOnEvaluation(
const ExperimentalFeature & feature,
const std::string_view fName,
const Pos & pos)
{
if (!settings.isExperimentalFeatureEnabled(feature)) {
throw EvalError({
.msg = hintfmt(
"Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
feature,
fName
),
.errPos = pos
});
}
}
void EvalState::allowPath(const Path & path)
{
if (allowedPaths)
@ -1903,13 +1893,22 @@ std::string_view EvalState::forceString(Value & v, const Pos & pos)
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<std::string, std::string> decodeContext(std::string_view s)
NixStringContextElem decodeContext(const Store & store, std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
return {
store.parseStorePath(s.substr(index + 1)),
std::string(s.substr(1, index - 1)),
};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
return {
store.parseStorePath(
s.at(0) == '/'
? s
: s.substr(1)),
"",
};
}
@ -1921,13 +1920,13 @@ void copyContext(const Value & v, PathSet & context)
}
std::vector<std::pair<Path, std::string>> Value::getContext()
NixStringContext Value::getContext(const Store & store)
{
std::vector<std::pair<Path, std::string>> res;
NixStringContext res;
assert(internalType == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(*p));
res.push_back(decodeContext(store, *p));
return res;
}

10
src/libexpr/eval.hh
View file

@ -78,7 +78,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sContentAddressed,
sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
@ -149,12 +149,6 @@ public:
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
void requireExperimentalFeatureOnEvaluation(
const ExperimentalFeature &,
const std::string_view fName,
const Pos & pos
);
void addToSearchPath(const std::string & s);
SearchPath getSearchPath() { return searchPath; }
@ -430,7 +424,7 @@ std::string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<std::string, std::string> decodeContext(std::string_view s);
NixStringContextElem decodeContext(const Store & store, std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);

27
src/libexpr/flake/flake.cc
View file

@ -708,8 +708,6 @@ void callFlake(EvalState & state,
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
std::string flakeRefS(state.forceStringNoCtx(*args[0], pos));
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
if (evalSettings.pureEval && !flakeRef.input.isLocked())
@ -725,7 +723,30 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
v);
}
static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
static RegisterPrimOp r2({
.name = "__getFlake",
.args = {"args"},
.doc = R"(
Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
```nix
(builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
```
Unless impure evaluation is allowed (`--impure`), the flake reference
must be "locked", e.g. contain a Git revision or content hash. An
example of an unlocked usage is:
```nix
(builtins.getFlake "github:edolstra/dwarffs").rev
```
This function is only available if you enable the experimental feature
`flakes`.
)",
.fun = prim_getFlake,
.experimentalFeature = Xp::Flakes,
});
}

23
src/libexpr/lexer.l
View file

@ -28,6 +28,13 @@ using namespace nix;
namespace nix {
static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
{
return Pos(data->origin, data->file, loc.first_line, loc.first_column);
}
#define CUR_POS makeCurPos(*yylloc, data)
// backup to recover from yyless(0)
YYLTYPE prev_yylloc;
@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc)
loc->first_column = loc->last_column = 1;
}
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
prev_yylloc = *loc;
@ -147,14 +153,20 @@ or { return OR_KW; }
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
throw ParseError("invalid integer '%1%'", yytext);
throw ParseError({
.msg = hintfmt("invalid integer '%1%'", yytext),
.errPos = CUR_POS,
});
}
return INT;
}
{FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0);
if (errno != 0)
throw ParseError("invalid float '%1%'", yytext);
throw ParseError({
.msg = hintfmt("invalid float '%1%'", yytext),
.errPos = CUR_POS,
});
return FLOAT;
}
@ -280,7 +292,10 @@ or { return OR_KW; }
<INPATH_SLASH>{ANY} |
<INPATH_SLASH><<EOF>> {
throw ParseError("path has a trailing slash");
throw ParseError({
.msg = hintfmt("path has a trailing slash"),
.errPos = CUR_POS,
});
}
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }

147
src/libexpr/primops.cc
View file

@ -43,8 +43,8 @@ StringMap EvalState::realiseContext(const PathSet & context)
StringMap res;
for (auto & i : context) {
auto [ctxS, outputName] = decodeContext(i);
auto ctx = store->parseStorePath(ctxS);
auto [ctx, outputName] = decodeContext(*store, i);
auto ctxS = store->printStorePath(ctx);
if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) {
@ -694,7 +694,32 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
.name = "__genericClosure",
.args = {"attrset"},
.arity = 1,
.doc = R"(
Take an *attrset* with values named `startSet` and `operator` in order to
return a *list of attrsets* by starting with the `startSet`, recursively
applying the `operator` function to each element. The *attrsets* in the
`startSet` and produced by the `operator` must each contain value named
`key` which are comparable to each other. The result is produced by
repeatedly calling the operator for each element encountered with a
unique key, terminating when no new elements are produced. For example,
```
builtins.genericClosure {
startSet = [ {key = 5;} ];
operator = item: [{
key = if (item.key / 2 ) * 2 == item.key
then item.key / 2
else 3 * item.key + 1;
}];
}
```
evaluates to
```
[ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
```
)",
.fun = prim_genericClosure,
});
@ -964,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
bool contentAddressed = false;
bool isImpure = false;
std::optional<std::string> outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
std::optional<FileIngestionMethod> ingestionMethod;
StringSet outputs;
outputs.insert("out");
@ -1026,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
settings.requireExperimentalFeature(Xp::CaDerivations);
}
else if (i->name == state.sImpure) {
isImpure = state.forceBool(*i->value, pos);
if (isImpure)
settings.requireExperimentalFeature(Xp::ImpureDerivations);
}
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
@ -1118,8 +1150,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Handle derivation outputs of the form ‘!<name>!<path>’. */
else if (path.at(0) == '!') {
auto ctx = decodeContext(path);
drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
auto ctx = decodeContext(*state.store, path);
drv.inputDrvs[ctx.first].insert(ctx.second);
}
/* Otherwise it's a source file. */
@ -1158,29 +1190,44 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
.errPos = posDrvName
});
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
Hash h = newHashAllowEmpty(*outputHash, ht);
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out",
DerivationOutput::CAFixed {
.hash = FixedOutputHash {
.method = ingestionMethod,
.method = method,
.hash = std::move(h),
},
});
}
else if (contentAddressed) {
HashType ht = parseHashType(outputHashAlgo);
else if (contentAddressed || isImpure) {
if (contentAddressed && isImpure)
throw EvalError({
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
.errPos = posDrvName
});
auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = ingestionMethod,
.hashType = ht,
});
if (isImpure)
drv.outputs.insert_or_assign(i,
DerivationOutput::Impure {
.method = method,
.hashType = ht,
});
else
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = method,
.hashType = ht,
});
}
}
@ -1197,34 +1244,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
DerivationOutput::Deferred { });
}
// Regular, non-CA derivation should always return a single hash and not
// hash per output.
auto hashModulo = hashDerivationModulo(*state.store, drv, true);
std::visit(overloaded {
[&](const DrvHash & drvHash) {
auto & h = drvHash.hash;
switch (drvHash.kind) {
case DrvHash::Kind::Deferred:
/* Outputs already deferred, nothing to do */
break;
case DrvHash::Kind::Regular:
for (auto & [outputName, output] : drv.outputs) {
auto outPath = state.store->makeOutputPath(outputName, h, drvName);
drv.env[outputName] = state.store->printStorePath(outPath);
output = DerivationOutput::InputAddressed {
.path = std::move(outPath),
};
}
break;
}
},
[&](const CaOutputHashes &) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
},
hashModulo.raw());
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = hashModulo.hashes.at(i);
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutputInputAddressed {
.path = std::move(outPath),
});
}
break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
}
}
}
/* Write the resulting term into the Nix store directory. */
@ -3789,7 +3828,7 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
.name = name,
.args = {},
.arity = arity,
.fun = fun
.fun = fun,
});
}
@ -3861,13 +3900,17 @@ void EvalState::createBaseEnv()
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
addPrimOp({
.fun = primOp.fun,
.arity = std::max(primOp.args.size(), primOp.arity),
.name = symbols.create(primOp.name),
.args = primOp.args,
.doc = primOp.doc,
});
if (!primOp.experimentalFeature
|| settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature))
{
addPrimOp({
.fun = primOp.fun,
.arity = std::max(primOp.args.size(), primOp.arity),
.name = symbols.create(primOp.name),
.args = primOp.args,
.doc = primOp.doc,
});
}
/* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily. */

2
src/libexpr/primops.hh
View file

@ -16,6 +16,7 @@ struct RegisterPrimOp
size_t arity = 0;
const char * doc;
PrimOpFun fun;
std::optional<ExperimentalFeature> experimentalFeature;
};
typedef std::vector<Info> PrimOps;
@ -35,6 +36,7 @@ struct RegisterPrimOp
/* These primops are disabled without enableNativeCode, but plugins
may wish to use them in limited contexts without globally enabling
them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);

4
src/libexpr/primops/context.cc
View file

@ -83,8 +83,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
drv = std::string(p, 1);
path = &drv;
} else if (p.at(0) == '!') {
std::pair<std::string, std::string> ctx = decodeContext(p);
drv = ctx.first;
NixStringContextElem ctx = decodeContext(*state.store, p);
drv = state.store->printStorePath(ctx.first);
output = ctx.second;
path = &drv;
}

161
src/libexpr/primops/fetchClosure.cc Normal file
View file

@ -0,0 +1,161 @@
#include "primops.hh"
#include "store-api.hh"
#include "make-content-addressed.hh"
#include "url.hh"
namespace nix {
static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos);
std::optional<std::string> fromStoreUrl;
std::optional<StorePath> fromPath;
bool toCA = false;
std::optional<StorePath> toPath;
for (auto & attr : *args[0]->attrs) {
if (attr.name == "fromPath") {
PathSet context;
fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
}
else if (attr.name == "toPath") {
state.forceValue(*attr.value, *attr.pos);
toCA = true;
if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
PathSet context;
toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
}
}
else if (attr.name == "fromStore")
fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw Error({
.msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
.errPos = pos
});
}
if (!fromPath)
throw Error({
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
.errPos = pos
});
if (!fromStoreUrl)
throw Error({
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
.errPos = pos
});
auto parsedURL = parseURL(*fromStoreUrl);
if (parsedURL.scheme != "http" &&
parsedURL.scheme != "https" &&
!(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
throw Error({
.msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
.errPos = pos
});
if (!parsedURL.query.empty())
throw Error({
.msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.errPos = pos
});
auto fromStore = openStore(parsedURL.to_string());
if (toCA) {
if (!toPath || !state.store->isValidPath(*toPath)) {
auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
auto i = remappings.find(*fromPath);
assert(i != remappings.end());
if (toPath && *toPath != i->second)
throw Error({
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second),
state.store->printStorePath(*toPath)),
.errPos = pos
});
if (!toPath)
throw Error({
.msg = hintfmt(
"rewriting '%s' to content-addressed form yielded '%s'; "
"please set this in the 'toPath' attribute passed to 'fetchClosure'",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second)),
.errPos = pos
});
}
} else {
if (!state.store->isValidPath(*fromPath))
copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
toPath = fromPath;
}
/* In pure mode, require a CA path. */
if (evalSettings.pureEval) {
auto info = state.store->queryPathInfo(*toPath);
if (!info->isContentAddressed(*state.store))
throw Error({
.msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
state.store->printStorePath(*toPath)),
.errPos = pos
});
}
auto toPathS = state.store->printStorePath(*toPath);
v.mkString(toPathS, {toPathS});
}
static RegisterPrimOp primop_fetchClosure({
.name = "__fetchClosure",
.args = {"args"},
.doc = R"(
Fetch a Nix store closure from a binary cache, rewriting it into
content-addressed form. For example,
```nix
builtins.fetchClosure {
fromStore = "https://cache.nixos.org";
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
}
```
fetches `/nix/store/r2jd...` from the specified binary cache,
and rewrites it into the content-addressed store path
`/nix/store/ldbh...`.
If `fromPath` is already content-addressed, or if you are
allowing impure evaluation (`--impure`), then `toPath` may be
omitted.
To find out the correct value for `toPath` given a `fromPath`,
you can use `nix store make-content-addressed`:
```console
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
```
This function is similar to `builtins.storePath` in that it
allows you to use a previously built store path in a Nix
expression. However, it is more reproducible because it requires
specifying a binary cache from which the path can be fetched.
Also, requiring a content-addressed final store path avoids the
need for users to configure binary cache public keys.
This function is only available if you enable the experimental
feature `fetch-closure`.
)",
.fun = prim_fetchClosure,
.experimentalFeature = Xp::FetchClosure,
});
}

4
src/libexpr/primops/fetchTree.cc
View file

@ -145,7 +145,7 @@ static void fetchTree(
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
.msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
.msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
.errPos = pos
});
@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({
.fun = prim_fetchTarball,
});
static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
}

3
src/libexpr/value-to-json.cc
View file

@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict,
.msg = hintfmt("cannot convert %1% to JSON", showType(v)),
.errPos = v.determinePos(pos)
});
throw e.addTrace(pos, hintfmt("message for the trace"));
e.addTrace(pos, hintfmt("message for the trace"));
throw e;
}
}

11
src/libexpr/value.hh
View file

@ -57,6 +57,8 @@ struct ExprLambda;
struct PrimOp;
class Symbol;
struct Pos;
class StorePath;
class Store;
class EvalState;
class XMLWriter;
class JSONPlaceholder;
@ -64,6 +66,8 @@ class JSONPlaceholder;
typedef int64_t NixInt;
typedef double NixFloat;
typedef std::pair<StorePath, std::string> NixStringContextElem;
typedef std::vector<NixStringContextElem> NixStringContext;
/* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented
@ -115,10 +119,13 @@ private:
InternalType internalType;
friend std::string showType(const Value & v);
friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v);
void print(std::ostream & str, std::set<const void *> * seen) const;
public:
void print(std::ostream & str, bool showRepeated = false) const;
// Functions needed to distinguish the type
// These should be removed eventually, by putting the functionality that's
// needed by callers into methods of this type
@ -368,7 +375,7 @@ public:
non-trivial. */
bool isTrivial() const;
std::vector<std::pair<Path, std::string>> getContext();
NixStringContext getContext(const Store &);
auto listItems()
{

15
src/libfetchers/fetchers.cc
View file

@ -238,9 +238,18 @@ std::optional<std::string> Input::getRef() const
std::optional<Hash> Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
return Hash::parseAny(*s, htSHA1);
return {};
std::optional<Hash> hash = {};
if (auto s = maybeGetStrAttr(attrs, "rev")) {
try {
hash = Hash::parseAnyPrefixed(*s);
} catch (BadHash &e) {
// Default to sha1 for backwards compatibility with existing flakes
hash = Hash::parseAny(*s, htSHA1);
}
}
return hash;
}
std::optional<uint64_t> Input::getRevCount() const

10
src/libfetchers/git.cc
View file

@ -285,9 +285,11 @@ struct GitInputScheme : InputScheme
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
Path actualPath(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
std::string file(p, actualUrl.size() + 1);
assert(hasPrefix(p, actualPath));
std::string file(p, actualPath.size() + 1);
auto st = lstat(p);
@ -300,13 +302,13 @@ struct GitInputScheme : InputScheme
return files.count(file);
};
auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
}

10
src/libfetchers/github.cc
View file

@ -390,7 +390,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
ref_uri = line.substr(ref_index+5, line.length()-1);
} else
ref_uri = fmt("refs/heads/%s", ref);
ref_uri = fmt("refs/(heads|tags)/%s", ref);
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
@ -399,9 +399,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string line;
std::string id;
while(getline(is, line)) {
auto index = line.find(ref_uri);
if (index != std::string::npos) {
id = line.substr(0, index-1);
// Append $ to avoid partial name matches
std::regex pattern(fmt("%s$", ref_uri));
if (std::regex_search(line, pattern)) {
id = line.substr(0, line.find('\t'));
break;
}
}

3
src/libstore/build-result.hh
View file

@ -1,6 +1,7 @@
#pragma once
#include "realisation.hh"
#include "derived-path.hh"
#include <string>
#include <chrono>
@ -30,6 +31,8 @@ struct BuildResult
ResolvesToAlreadyValid,
NoSubstituters,
} status = MiscFailure;
// FIXME: include entire ErrorInfo object.
std::string errorMsg;
std::string toString() const {

194
src/libstore/build/derivation-goal.cc
View file

@ -204,10 +204,33 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
retrySubstitution = false;
if (!drv->type().isPure()) {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
assert(!worker.store.isValidPath(randomPath));
initialOutputs.insert({
outputName,
InitialOutput {
.wanted = true,
.outputHash = impureOutputHash,
.known = InitialOutputStatus {
.path = randomPath,
.status = PathStatus::Absent
}
}
});
}
gaveUpOnSubstitution();
return;
}
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
@ -232,9 +255,6 @@ void DerivationGoal::haveDerivation()
return;
}
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@ -268,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
assert(drv->type().isPure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@ -311,18 +333,27 @@ void DerivationGoal::outputsSubstitutionTried()
gaveUpOnSubstitution();
}
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
void DerivationGoal::gaveUpOnSubstitution()
{
/* Make sure checkPathValidity() from now on checks all
outputs. */
wantedOutputs.clear();
/* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (drv->type().isPure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(i.first));
}
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
}
/* Copy the input sources from the eval store to the build
store. */
@ -350,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure()
{
assert(drv->type().isPure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
@ -426,7 +459,8 @@ void DerivationGoal::inputsRealised()
return;
}
if (retrySubstitution) {
if (retrySubstitution && !retriedSubstitution) {
retriedSubstitution = true;
haveDerivation();
return;
}
@ -456,22 +490,24 @@ void DerivationGoal::inputsRealised()
drvs. */
: true);
},
[&](const DerivationType::Impure &) {
return true;
}
}, drvType.raw());
if (resolveDrv)
{
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
settings.requireExperimentalFeature(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
std::optional attempt = fullDrv.tryResolve(worker.store);
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
assert(attempt);
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
auto msg = fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@ -492,21 +528,13 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);
if (!optRealizedInput)
throw Error(
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
} else
for (auto & j : wantedDepOutputs)
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
worker.store.computeFSClosure(*outPath, inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
}
}
@ -928,7 +956,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
!derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@ -949,60 +977,53 @@ void DerivationGoal::buildDone()
void DerivationGoal::resolvedFinished()
{
trace("resolved derivation finished");
assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv;
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
);
// We've just built it, but maybe the build failed, in which case the
// realisation won't be there
if (realisation) {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
assert(!buildResult.success());
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = resolvedResult.builtOutputs.at(
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
if (drv->type().isPure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
newRealisation.signatures.clear();
if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
auto status = [&]() {
auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) {
case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid;
default:
return resolvedResult.status;
}
}();
auto status = resolvedResult.status;
if (status == BuildResult::AlreadyValid)
status = BuildResult::ResolvesToAlreadyValid;
done(status, std::move(builtOutputs));
}
@ -1251,6 +1272,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
@ -1263,6 +1285,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@ -1276,6 +1299,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{
if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
@ -1319,6 +1344,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
@ -1356,9 +1382,7 @@ void DerivationGoal::done(
{
buildResult.status = status;
if (ex)
// FIXME: strip: "error: "
buildResult.errorMsg = ex->what();
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)
@ -1385,7 +1409,21 @@ void DerivationGoal::done(
fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
}
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
}
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success())
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName },
realisation.outPath);
}
}

15
src/libstore/build/derivation-goal.hh
View file

@ -57,12 +57,21 @@ struct DerivationGoal : public Goal
them. */
StringSet wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
inputs. */
bool retrySubstitution;
inputs. This is done in case of an incomplete closure. */
bool retrySubstitution = false;
/* Whether we've retried substitution, in which case we won't try
again. */
bool retriedSubstitution = false;
/* The derivation stored at drvPath. */
std::unique_ptr<Derivation> drv;
@ -225,6 +234,8 @@ struct DerivationGoal : public Goal
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
};

2
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -41,7 +41,7 @@ void DrvOutputSubstitutionGoal::tryNext()
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a

2
src/libstore/build/goal.cc
View file

@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
{
assert(waitees.find(waitee) != waitees.end());
assert(waitees.count(waitee));
waitees.erase(waitee);
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));

13
src/libstore/build/goal.hh
View file

@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this<Goal>
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed;
size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
unsigned int nrNoSubstituters;
size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
unsigned int nrIncompleteClosure;
size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */
std::string name;
/* Whether the goal is finished. */
ExitCode exitCode;
ExitCode exitCode = ecBusy;
/* Build result. */
BuildResult buildResult;
@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
{
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy;
}
{ }
virtual ~Goal()
{

29
src/libstore/build/local-derivation-goal.cc
View file

@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = !(derivationType.isImpure()) && !noChroot;
useChroot = derivationType.isSandboxed() && !noChroot;
}
auto & localStore = getLocalStore();
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
if (!(derivationType.isImpure()))
if (derivationType.isSandboxed())
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@ -704,6 +704,9 @@ void LocalDerivationGoal::startBuilder()
/* Run the builder. */
printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args));
for (auto & i : drv->env)
printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second);
/* Create the log file. */
Path logFile = openLogFile();
@ -797,7 +800,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
if (!(derivationType.isImpure()))
if (derivationType.isSandboxed())
privateNetwork = true;
userNamespaceSync.create();
@ -1061,7 +1064,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
if (derivationType.isImpure()) {
if (!derivationType.isSandboxed()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@ -1675,7 +1678,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
if (derivationType.isImpure()) {
if (!derivationType.isSandboxed()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@ -1919,7 +1922,7 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
if (derivationType.isImpure())
if (!derivationType.isSandboxed())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Add the output paths we'll use at build-time to the chroot */
@ -2400,6 +2403,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
assert(false);
},
[&](const DerivationOutput::Impure & doi) {
return newInfoFromCA(DerivationOutput::CAFloating {
.method = doi.method,
.hashType = doi.hashType,
});
},
}, output.raw());
/* FIXME: set proper permissions in restorePath() so
@ -2610,11 +2620,14 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
},
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
&& drv->type().isPure())
{
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
builtOutputs.emplace(thisRealisation.id, thisRealisation);
if (wantOutput(outputName, wantedOutputs))
builtOutputs.emplace(thisRealisation.id, thisRealisation);
}
return builtOutputs;

19
src/libstore/build/substitution-goal.cc
View file

@ -24,9 +24,16 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
}
void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status)
void PathSubstitutionGoal::done(
ExitCode result,
BuildResult::Status status,
std::optional<std::string> errorMsg)
{
buildResult.status = status;
if (errorMsg) {
debug(*errorMsg);
buildResult.errorMsg = *errorMsg;
}
amDone(result);
}
@ -67,12 +74,14 @@ void PathSubstitutionGoal::tryNext()
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath));
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters);
done(
substituterFailed ? ecFailed : ecNoSubstituters,
BuildResult::NoSubstituters,
fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)));
if (substituterFailed) {
worker.failedSubstitutions++;
@ -169,10 +178,10 @@ void PathSubstitutionGoal::referencesValid()
trace("all references realised");
if (nrFailed > 0) {
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
done(
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
BuildResult::DependencyFailed);
BuildResult::DependencyFailed,
fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)));
return;
}

5
src/libstore/build/substitution-goal.hh
View file

@ -56,7 +56,10 @@ struct PathSubstitutionGoal : public Goal
/* Time substitution started. */
std::chrono::time_point<std::chrono::steady_clock> startTime;
void done(ExitCode result, BuildResult::Status status);
void done(
ExitCode result,
BuildResult::Status status,
std::optional<std::string> errorMsg = {});
public:
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);

10
src/libstore/builtins/buildenv.cc
View file

@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw;
}
/* The files below are special-cased to that they don't show up
* in user profiles, either because they are useless, or
* because they would cauase pointless collisions (e.g., each
/* The files below are special-cased to that they don't show
* up in user profiles, either because they are useless, or
* because they would cause pointless collisions (e.g., each
* Python package brings its own
* `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
*/
@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") ||
hasSuffix(srcFile, "/log"))
hasSuffix(srcFile, "/log") ||
hasSuffix(srcFile, "/manifest.nix") ||
hasSuffix(srcFile, "/manifest.json"))
continue;
else if (S_ISDIR(srcSt.st_mode)) {

316
src/libstore/derivations.cc
View file

@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
return std::nullopt;
},
}, raw());
}
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool DerivationType::isCA() const {
bool DerivationType::isCA() const
{
/* Normally we do the full `std::visit` to make sure we have
exhaustively handled all variants, but so long as there is a
variant called `ContentAddressed`, it must be the only one for
which `isCA` is true for this to make sense!. */
return std::holds_alternative<ContentAddressed>(raw());
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return true;
},
}, raw());
}
bool DerivationType::isFixed() const {
bool DerivationType::isFixed() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::hasKnownOutputPaths() const {
bool DerivationType::hasKnownOutputPaths() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return !ia.deferred;
@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isImpure() const {
bool DerivationType::isSandboxed() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
return true;
},
[](const ContentAddressed & ca) {
return !ca.pure;
return ca.sandboxed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isPure() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return true;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return false;
},
}, raw());
}
@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
if (hash == "impure") {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
assert(pathS == "");
return DerivationOutput::Impure {
.method = std::move(method),
.hashType = std::move(hashType),
};
} else if (hash != "") {
validatePath(pathS);
return DerivationOutput::CAFixed {
.hash = FixedOutputHash {
@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutputImpure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw());
s += ')';
@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::set<std::string_view>
inputAddressedOutputs,
fixedCAOutputs,
floatingCAOutputs,
deferredIAOutputs,
impureOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed &) {
@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type");
throw Error("all floating outputs must use the same hash type");
}
},
[&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first);
deferredIAOutputs.insert(i.first);
},
[&](const DerivationOutput::Impure &) {
impureOutputs.insert(i.first);
},
}, i.second.raw());
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
throw Error("must have at least one output");
if (!inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = false,
};
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& !fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
{
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed {
.pure = false,
.sandboxed = false,
.fixed = true,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& !floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::ContentAddressed {
.pure = true,
.sandboxed = true,
.fixed = false,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& !deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = true,
};
} else {
throw Error("Can't mix derivation output types");
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& !impureOutputs.empty())
return DerivationType::Impure { };
throw Error("can't mix derivation output types");
}
@ -474,7 +571,7 @@ Sync<DrvHashes> drvHashes;
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
{
{
auto hashes = drvHashes.lock();
@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
don't leak the provenance of fixed outputs, reducing pointless cache
misses as the build itself won't know this.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
auto type = drv.type();
@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
}
if (!type.isPure()) {
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs)
outputHashes.insert_or_assign(outputName, impureOutputHash);
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Deferred,
};
}
auto kind = std::visit(overloaded {
@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
? DrvHash::Kind::Regular
: DrvHash::Kind::Deferred;
},
[](const DerivationType::Impure &) -> DrvHash::Kind {
assert(false);
}
}, drv.type().raw());
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2;
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
// Avoid lambda capture restriction with standard / Clang
auto & inputOutputs = inputOutputs0;
const auto & res = pathDerivationModulo(store, drvPath);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
[&](const DrvHash & drvHash) {
kind |= drvHash.kind;
inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
},
// CA derivation's output hashes
[&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" };
for (auto & output : inputOutputs) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
h.to_string(Base16, false),
justOut);
}
},
}, res.raw());
if (res.kind == DrvHash::Kind::Deferred)
kind = DrvHash::Kind::Deferred;
for (auto & outputName : inputOutputs) {
const auto h = res.hashes.at(outputName);
inputs2[h.to_string(Base16, false)].insert(outputName);
}
}
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
return DrvHash { .hash = hash, .kind = kind };
}
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
{
switch (other) {
case DrvHash::Kind::Regular:
break;
case DrvHash::Kind::Deferred:
self = other;
break;
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs) {
outputHashes.insert_or_assign(outputName, hash);
}
return DrvHash {
.hashes = outputHashes,
.kind = kind,
};
}
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
{
std::map<std::string, Hash> res;
std::visit(overloaded {
[&](const DrvHash & drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash.hash});
}
},
[&](const CaOutputHashes & outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true).raw());
return res;
return hashDerivationModulo(store, drv, true).hashes;
}
@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
{
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
return outsAndOptPaths;
}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
{
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< ""
<< "";
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
<< "impure";
},
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
@ -723,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
for (auto &rewrite: rewrites) {
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
{
for (auto & rewrite : rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
for (auto & arg: drv.args) {
for (auto & arg : drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
for (auto & envVar: drv.env) {
for (auto & envVar : drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
auto & h = hashModulo.requireNoFixedNonDeferred();
auto & h = hashModulo.hashes.at(outputName);
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput::InputAddressed {
@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
auto * drvHashOpt = std::get_if<DrvHash>(&raw());
assert(drvHashOpt);
assert(drvHashOpt->kind == DrvHash::Kind::Regular);
return drvHashOpt->hash;
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const StorePath & inputDrv, const StringSet & inputOutputs)
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
{
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
auto getOutput = [&](const std::string & outputName) {
auto & actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt)
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
for (auto & input : inputDrvs)
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
if (outputPath)
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
for (auto & outputName : inputOutputs) {
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(actualPath));
inputSrcs.insert(std::move(actualPath));
}
return true;
return tryResolve(store, inputDrvOutputs);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
std::optional<BasicDerivation> Derivation::tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
{
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
for (auto & [inputDrv, inputOutputs] : inputDrvs)
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
return std::nullopt;
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {
warn("output '%s' of input '%s' missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv));
return {};
}
}
}
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
const Hash impureOutputHash = hashString(htSHA256, "impure");
}

95
src/libstore/derivations.hh
View file

@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
* known yet.
*/
struct DerivationOutputDeferred {};
/* Impure output which is moved to a content-addressed location (like
CAFloating) but isn't registered as a realization.
*/
struct DerivationOutputImpure
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
typedef std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating,
DerivationOutputDeferred
DerivationOutputDeferred,
DerivationOutputImpure
> _DerivationOutputRaw;
struct DerivationOutput : _DerivationOutputRaw
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
using CAFixed = DerivationOutputCAFixed;
using CAFloating = DerivationOutputCAFloating;
using Deferred = DerivationOutputDeferred;
using Impure = DerivationOutputImpure;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
};
struct DerivationType_ContentAddressed {
bool pure;
bool sandboxed;
bool fixed;
};
struct DerivationType_Impure {
};
typedef std::variant<
DerivationType_InputAddressed,
DerivationType_ContentAddressed
DerivationType_ContentAddressed,
DerivationType_Impure
> _DerivationTypeRaw;
struct DerivationType : _DerivationTypeRaw {
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
using Raw::Raw;
using InputAddressed = DerivationType_InputAddressed;
using ContentAddressed = DerivationType_ContentAddressed;
using Impure = DerivationType_Impure;
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
non-CA derivations. */
bool isFixed() const;
/* Is the derivation impure and needs to access non-deterministic resources, or
pure and can be sandboxed? Note that whether or not we actually sandbox the
derivation is controlled separately. Never true for non-CA derivations. */
bool isImpure() const;
/* Whether the derivation is fully sandboxed. If false, the
sandbox is opened up, e.g. the derivation has access to the
network. Note that whether or not we actually sandbox the
derivation is controlled separately. Always true for non-CA
derivations. */
bool isSandboxed() const;
/* Whether the derivation is expected to produce the same result
every time, and therefore it only needs to be built once. This
is only false for derivations that have the attribute '__impure
= true'. */
bool isPure() const;
/* Does the derivation knows its own output paths?
Only true when there's no floating-ca derivation involved in the
@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
std::optional<BasicDerivation> tryResolve(Store & store) const;
/* Like the above, but instead of querying the Nix database for
realisations, uses a given mapping from input derivation paths
+ output names to actual output store paths. */
std::optional<BasicDerivation> tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@ -202,14 +233,16 @@ bool isDerivation(const std::string & fileName);
the output name is "out". */
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
// The hashes modulo of a derivation.
//
// Each output is given a hash, although in practice only the content-addressed
// derivations (fixed-output or not) will have a different hash for each
// output.
struct DrvHash {
Hash hash;
std::map<std::string, Hash> hashes;
enum struct Kind: bool {
enum struct Kind : bool {
// Statically determined derivations.
// This hash will be directly used to compute the output paths
Regular,
@ -222,28 +255,6 @@ struct DrvHash {
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
typedef std::variant<
// Regular normalized derivation hash, and whether it was deferred (because
// an ancestor derivation is a floating content addressed derivation).
DrvHash,
// Fixed-output derivation hashes
CaOutputHashes
> _DrvHashModuloRaw;
struct DrvHashModulo : _DrvHashModuloRaw {
using Raw = _DrvHashModuloRaw;
using Raw::Raw;
/* Get hash, throwing if it is per-output CA hashes or a
deferred Drv hash.
*/
const Hash & requireNoFixedNonDeferred() const;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
};
/* Returns hashes with the details of fixed-output subderivations
expunged.
@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw {
ATerm, after subderivations have been likewise expunged from that
derivation.
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
FIXME: what is the Hash in this map?
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
typedef std::map<StorePath, DrvHash> DrvHashes;
// FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes;
@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
}

6
src/libstore/derived-path.hh
View file

@ -25,6 +25,9 @@ struct DerivedPathOpaque {
nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const;
static DerivedPathOpaque parse(const Store & store, std::string_view);
bool operator < (const DerivedPathOpaque & b) const
{ return path < b.path; }
};
/**
@ -46,6 +49,9 @@ struct DerivedPathBuilt {
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view);
nlohmann::json toJSON(ref<Store> store) const;
bool operator < (const DerivedPathBuilt & b) const
{ return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); }
};
using _DerivedPathRaw = std::variant<

2
src/libstore/filetransfer.hh
View file

@ -123,8 +123,6 @@ public:
template<typename... Args>
FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args);
virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(std::string_view s);

12
src/libstore/local-store.cc
View file

@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
// combinations that are currently prohibited.
drv.type();
std::optional<Hash> h;
std::optional<DrvHash> hashesModulo;
for (auto & i : drv.outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed & doia) {
if (!h) {
if (!hashesModulo) {
// somewhat expensive so we do lazily
auto h0 = hashDerivationModulo(*this, drv, true);
h = h0.requireNoFixedNonDeferred();
hashesModulo = hashDerivationModulo(*this, drv, true);
}
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
if (doia.path != recomputed)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw());
}
}

80
src/libstore/make-content-addressed.cc Normal file
View file

@ -0,0 +1,80 @@
#include "make-content-addressed.hh"
#include "references.hh"
namespace nix {
std::map<StorePath, StorePath> makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths)
{
StorePathSet closure;
srcStore.computeFSClosure(storePaths, closure);
auto paths = srcStore.topoSortPaths(closure);
std::reverse(paths.begin(), paths.end());
std::map<StorePath, StorePath> remappings;
for (auto & path : paths) {
auto pathS = srcStore.printStorePath(path);
auto oldInfo = srcStore.queryPathInfo(path);
std::string oldHashPart(path.hashPart());
StringSink sink;
srcStore.narFromPath(path, sink);
StringMap rewrites;
StorePathSet references;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
references.insert(std::move(replacement));
}
}
sink.s = rewriteStrings(sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink(sink.s);
auto narModuloHash = hashModuloSink.finish().first;
auto dstPath = dstStore.makeFixedOutputPath(
FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
StringSink sink2;
RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
rsink2(sink.s);
rsink2.flush();
ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
};
StringSource source(sink2.s);
dstStore.addToStore(info, source);
remappings.insert_or_assign(std::move(path), std::move(info.path));
}
return remappings;
}
}

12
src/libstore/make-content-addressed.hh Normal file
View file

@ -0,0 +1,12 @@
#pragma once
#include "store-api.hh"
namespace nix {
std::map<StorePath, StorePath> makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths);
}

7
src/libstore/misc.cc
View file

@ -277,15 +277,15 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
{
std::set<Realisation> inputRealisations;
for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto& outputName : outputNames) {
for (const auto & outputName : outputNames) {
auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName,
"output '%s' of derivation '%s' isn't built", outputName,
store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation);
}
@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
}

9
src/libstore/path.cc
View file

@ -1,5 +1,7 @@
#include "store-api.hh"
#include <sodium.h>
namespace nix {
static void checkName(std::string_view path, std::string_view name)
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath StorePath::random(std::string_view name)
{
Hash hash(htSHA1);
randombytes_buf(hash.hash, hash.hashSize);
return StorePath(hash, name);
}
StorePath Store::parseStorePath(std::string_view path) const
{
auto p = canonPath(std::string(path));

2
src/libstore/path.hh
View file

@ -58,6 +58,8 @@ public:
}
static StorePath dummy;
static StorePath random(std::string_view name);
};
typedef std::set<StorePath> StorePathSet;

1
src/libstore/sqlite.cc
View file

@ -215,7 +215,6 @@ void handleSQLiteBusy(const SQLiteBusy & e)
if (now > lastWarned + 10) {
lastWarned = now;
logWarning({
.name = "Sqlite busy",
.msg = hintfmt(e.what())
});
}

6
src/libutil/error.cc
View file

@ -9,10 +9,9 @@ namespace nix {
const std::string nativeSystem = SYSTEM;
BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
{
err.traces.push_front(Trace { .pos = e, .hint = hint });
return *this;
}
// c++ std::exception descendants must have a 'const char* what()' function.
@ -22,12 +21,9 @@ const std::string & BaseError::calcWhat() const
if (what_.has_value())
return *what_;
else {
err.name = sname();
std::ostringstream oss;
showErrorInfo(oss, err, loggerSettings.showTrace);
what_ = oss.str();
return *what_;
}
}

12
src/libutil/error.hh
View file

@ -109,7 +109,6 @@ struct Trace {
struct ErrorInfo {
Verbosity level;
std::string name; // FIXME: rename
hintformat msg;
std::optional<ErrPos> errPos;
std::list<Trace> traces;
@ -162,8 +161,6 @@ public:
: err(e)
{ }
virtual const char* sname() const { return "BaseError"; }
#ifdef EXCEPTION_NEEDS_THROW_SPEC
~BaseError() throw () { };
const char * what() const throw () { return calcWhat().c_str(); }
@ -175,12 +172,12 @@ public:
const ErrorInfo & info() const { calcWhat(); return err; }
template<typename... Args>
BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
{
return addTrace(e, hintfmt(fs, args...));
addTrace(e, hintfmt(fs, args...));
}
BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
void addTrace(std::optional<ErrPos> e, hintformat hint);
bool hasTrace() const { return !err.traces.empty(); }
};
@ -190,7 +187,6 @@ public:
{ \
public: \
using superClass::superClass; \
virtual const char* sname() const override { return #newClass; } \
}
MakeError(Error, BaseError);
@ -210,8 +206,6 @@ public:
auto hf = hintfmt(args...);
err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}
virtual const char* sname() const override { return "SysError"; }
};
}

2
src/libutil/experimental-features.cc
View file

@ -7,10 +7,12 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
{ Xp::FetchClosure, "fetch-closure" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)

8
src/libutil/experimental-features.hh
View file

@ -16,10 +16,12 @@ namespace nix {
enum struct ExperimentalFeature
{
CaDerivations,
ImpureDerivations,
Flakes,
NixCommand,
RecursiveNix,
NoUrlLiterals
NoUrlLiterals,
FetchClosure,
};
/**
@ -47,10 +49,6 @@ public:
ExperimentalFeature missingFeature;
MissingExperimentalFeature(ExperimentalFeature);
virtual const char * sname() const override
{
return "MissingExperimentalFeature";
}
};
}

2
src/libutil/hash.cc
View file

@ -155,7 +155,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_
{
bool isSRI = false;
// Parse the has type before the separater, if there was one.
// Parse the hash type before the separator, if there was one.
std::optional<HashType> optParsedType;
{
auto hashRaw = splitPrefixTo(rest, ':');

2
src/libutil/hash.hh
View file

@ -93,13 +93,11 @@ public:
std::string gitRev() const
{
assert(type == htSHA1);
return to_string(Base16, false);
}
std::string gitShortRev() const
{
assert(type == htSHA1);
return std::string(to_string(Base16, false), 0, 7);
}

5
src/libutil/serialise.cc
View file

@ -357,7 +357,7 @@ Sink & operator << (Sink & sink, const Error & ex)
sink
<< "Error"
<< info.level
<< info.name
<< "Error" // removed
<< info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
@ -426,11 +426,10 @@ Error readError(Source & source)
auto type = readString(source);
assert(type == "Error");
auto level = (Verbosity) readInt(source);
auto name = readString(source);
auto name = readString(source); // removed
auto msg = readString(source);
ErrorInfo info {
.level = level,
.name = name,
.msg = hintformat(std::move(format("%s") % msg)),
};
auto havePos = readNum<size_t>(source);

26
src/libutil/tarfile.cc
View file

@ -39,30 +39,32 @@ void TarArchive::check(int err, const std::string & reason)
throw Error(reason, archive_error_string(this->archive));
}
TarArchive::TarArchive(Source & source, bool raw)
: source(&source), buffer(4096)
TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
{
init();
if (!raw)
this->archive = archive_read_new();
this->source = &source;
if (!raw) {
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
else
} else {
archive_read_support_filter_all(archive);
archive_read_support_format_raw(archive);
archive_read_support_format_empty(archive);
}
check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
}
TarArchive::TarArchive(const Path & path)
{
init();
this->archive = archive_read_new();
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
void TarArchive::init()
{
archive = archive_read_new();
archive_read_support_filter_all(archive);
}
void TarArchive::close()
{
check(archive_read_close(this->archive), "Failed to close archive (%s)");

3
src/libutil/tarfile.hh
View file

@ -17,13 +17,10 @@ struct TarArchive {
// disable copy constructor
TarArchive(const TarArchive &) = delete;
void init();
void close();
~TarArchive();
};
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);

21
src/libutil/util.cc
View file

@ -71,13 +71,11 @@ void clearEnv()
unsetenv(name.first.c_str());
}
void replaceEnv(std::map<std::string, std::string> newEnv)
void replaceEnv(const std::map<std::string, std::string> & newEnv)
{
clearEnv();
for (auto newEnvVar : newEnv)
{
for (auto & newEnvVar : newEnv)
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
}
}
@ -1261,9 +1259,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato
{
C result;
auto pos = s.find_first_not_of(separators, 0);
while (pos != std::string::npos) {
while (pos != std::string_view::npos) {
auto end = s.find_first_of(separators, pos + 1);
if (end == std::string::npos) end = s.size();
if (end == std::string_view::npos) end = s.size();
result.insert(result.end(), std::string(s, pos, end - pos));
pos = s.find_first_not_of(separators, end);
}
@ -1473,6 +1471,7 @@ constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv
std::string base64Encode(std::string_view s)
{
std::string res;
res.reserve((s.size() + 2) / 3 * 4);
int data = 0, nbits = 0;
for (char c : s) {
@ -1504,6 +1503,9 @@ std::string base64Decode(std::string_view s)
}();
std::string res;
// Some sequences are missing the padding consisting of up to two '='.
// vvv
res.reserve((s.size() + 2) / 4 * 3);
unsigned int d = 0, bits = 0;
for (char c : s) {
@ -1690,7 +1692,9 @@ void setStackSize(size_t stackSize)
#endif
}
#if __linux__
static AutoCloseFD fdSavedMountNamespace;
#endif
void saveMountNamespace()
{
@ -1709,8 +1713,13 @@ void restoreMountNamespace()
{
#if __linux__
try {
auto savedCwd = absPath(".");
if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
throw SysError("restoring parent mount namespace");
if (chdir(savedCwd.c_str()) == -1) {
throw SysError("restoring cwd");
}
} catch (Error & e) {
debug(e.msg());
}

4
src/nix-env/user-env.cc
View file

@ -105,8 +105,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
/* Also write a copy of the list of user environment elements to
the store; we need it for future modifications of the
environment. */
std::ostringstream str;
manifest.print(str, true);
auto manifestFile = state.store->addTextToStore("env-manifest.nix",
fmt("%s", manifest), references);
str.str(), references);
/* Get the environment builder expression. */
Value envBuilder;

2
src/nix/app.cc
View file

@ -71,7 +71,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
std::vector<StorePathWithOutputs> context2;
for (auto & [path, name] : context)
context2.push_back({state.store->parseStorePath(path), {name}});
context2.push_back({path, {name}});
return UnresolvedApp{App {
.context = std::move(context2),

2
src/nix/bundle.cc
View file

@ -9,7 +9,7 @@ using namespace nix;
struct CmdBundle : InstallableCommand
{
std::string bundler = "github:matthewbauer/nix-bundle";
std::string bundler = "github:NixOS/bundlers";
std::optional<Path> outLink;
CmdBundle()

4
src/nix/develop.cc
View file

@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
output.second = DerivationOutput::Deferred { };
drv.env[output.first] = "";
}
auto h0 = hashDerivationModulo(*evalStore, drv, true);
const Hash & h = h0.requireNoFixedNonDeferred();
auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
for (auto & output : drv.outputs) {
Hash h = hashesModulo.hashes.at(output.first);
auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = DerivationOutput::InputAddressed {
.path = outPath,

4
src/nix/doctor.cc
View file

@ -24,12 +24,12 @@ std::string formatProtocol(unsigned int proto)
}
bool checkPass(const std::string & msg) {
logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg);
notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg);
return true;
}
bool checkFail(const std::string & msg) {
logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg);
notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg);
return false;
}

6
src/nix/edit.md
View file

@ -24,8 +24,8 @@ this attribute to the location of the definition of the
`meta.description`, `version` or `name` derivation attributes.
The editor to invoke is specified by the `EDITOR` environment
variable. It defaults to `cat`. If the editor is `emacs`, `nano` or
`vim`, it is passed the line number of the derivation using the
argument `+<lineno>`.
variable. It defaults to `cat`. If the editor is `emacs`, `nano`,
`vim` or `kak`, it is passed the line number of the derivation using
the argument `+<lineno>`.
)""

3
src/nix/flake.cc
View file

@ -463,7 +463,7 @@ struct CmdFlakeCheck : FlakeCommand
for (auto & attr : *v.attrs) {
std::string name(attr.name);
if (name != "path" && name != "description")
if (name != "path" && name != "description" && name != "welcomeText")
throw Error("template '%s' has unsupported attribute '%s'", attrPath, name);
}
} catch (Error & e) {
@ -508,6 +508,7 @@ struct CmdFlakeCheck : FlakeCommand
name == "defaultBundler" ? "bundlers.<system>.default" :
name == "overlay" ? "overlays.default" :
name == "devShell" ? "devShells.<system>.default" :
name == "nixosModule" ? "nixosModules.default" :
"";
if (replacement != "")
warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement);

3
src/nix/main.cc
View file

@ -118,7 +118,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{"hash-path", {"hash", "path"}},
{"ls-nar", {"nar", "ls"}},
{"ls-store", {"store", "ls"}},
{"make-content-addressable", {"store", "make-content-addressable"}},
{"make-content-addressable", {"store", "make-content-addressed"}},
{"optimise-store", {"store", "optimise"}},
{"ping-store", {"store", "ping"}},
{"sign-paths", {"store", "sign"}},
@ -287,6 +287,7 @@ void mainWrapped(int argc, char * * argv)
}
if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure};
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://"));
auto res = nlohmann::json::object();

102
src/nix/make-content-addressable.cc
View file

@ -1,102 +0,0 @@
#include "command.hh"
#include "store-api.hh"
#include "references.hh"
#include "common-args.hh"
#include "json.hh"
using namespace nix;
struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
{
CmdMakeContentAddressable()
{
realiseMode = Realise::Outputs;
}
std::string description() override
{
return "rewrite a path or closure to content-addressed form";
}
std::string doc() override
{
return
#include "make-content-addressable.md"
;
}
void run(ref<Store> store, StorePaths && storePaths) override
{
auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
std::reverse(paths.begin(), paths.end());
std::map<StorePath, StorePath> remappings;
auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
for (auto & path : paths) {
auto pathS = store->printStorePath(path);
auto oldInfo = store->queryPathInfo(path);
std::string oldHashPart(path.hashPart());
StringSink sink;
store->narFromPath(path, sink);
StringMap rewrites;
StorePathSet references;
bool hasSelfReference = false;
for (auto & ref : oldInfo->references) {
if (ref == path)
hasSelfReference = true;
else {
auto i = remappings.find(ref);
auto replacement = i != remappings.end() ? i->second : ref;
// FIXME: warn about unremapped paths?
if (replacement != ref)
rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
references.insert(std::move(replacement));
}
}
sink.s = rewriteStrings(sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
hashModuloSink(sink.s);
auto narHash = hashModuloSink.finish().first;
ValidPathInfo info {
store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
narHash,
};
info.references = std::move(references);
if (hasSelfReference) info.references.insert(info.path);
info.narSize = sink.s.size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = info.narHash,
};
if (!json)
notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
rsink2(sink.s);
rsink2.flush();
});
store->addToStore(info, *source);
if (json)
jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
remappings.insert_or_assign(std::move(path), std::move(info.path));
}
}
};
static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});

55
src/nix/make-content-addressed.cc Normal file
View file

@ -0,0 +1,55 @@
#include "command.hh"
#include "store-api.hh"
#include "make-content-addressed.hh"
#include "common-args.hh"
#include "json.hh"
using namespace nix;
struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
{
CmdMakeContentAddressed()
{
realiseMode = Realise::Outputs;
}
std::string description() override
{
return "rewrite a path or closure to content-addressed form";
}
std::string doc() override
{
return
#include "make-content-addressed.md"
;
}
void run(ref<Store> srcStore, StorePaths && storePaths) override
{
auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
auto remappings = makeContentAddressed(*srcStore, *dstStore,
StorePathSet(storePaths.begin(), storePaths.end()));
if (json) {
JSONObject jsonRoot(std::cout);
JSONObject jsonRewrites(jsonRoot.object("rewrites"));
for (auto & path : storePaths) {
auto i = remappings.find(path);
assert(i != remappings.end());
jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
}
} else {
for (auto & path : storePaths) {
auto i = remappings.find(path);
assert(i != remappings.end());
notice("rewrote '%s' to '%s'",
srcStore->printStorePath(path),
srcStore->printStorePath(i->second));
}
}
}
};
static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});

4
src/nix/make-content-addressable.md → src/nix/make-content-addressed.md
View file

@ -5,7 +5,7 @@ R""(
* Create a content-addressed representation of the closure of GNU Hello:
```console
# nix store make-content-addressable -r nixpkgs#hello
# nix store make-content-addressed nixpkgs#hello
…
rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
```
@ -29,7 +29,7 @@ R""(
system closure:
```console
# nix store make-content-addressable -r /run/current-system
# nix store make-content-addressed /run/current-system
```
# Description

63
src/nix/profile.cc
View file

@ -62,22 +62,21 @@ struct ProfileElement
return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
}
void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable)
void updateStorePaths(
ref<Store> evalStore,
ref<Store> store,
const BuiltPaths & builtPaths)
{
// FIXME: respect meta.outputsToInstall
storePaths.clear();
for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) {
for (auto & buildable : builtPaths) {
std::visit(overloaded {
[&](const BuiltPath::Opaque & bo) {
storePaths.insert(bo.path);
},
[&](const BuiltPath::Built & bfd) {
// TODO: Why are we querying if we know the output
// names already? Is it just to figure out what the
// default one is?
for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
for (auto & output : bfd.outputs)
storePaths.insert(output.second);
}
},
}, buildable.raw());
}
@ -98,18 +97,30 @@ struct ProfileManifest
auto json = nlohmann::json::parse(readFile(manifestPath));
auto version = json.value("version", 0);
if (version != 1)
throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version);
std::string sUrl;
std::string sOriginalUrl;
switch(version){
case 1:
sUrl = "uri";
sOriginalUrl = "originalUri";
break;
case 2:
sUrl = "url";
sOriginalUrl = "originalUrl";
break;
default:
throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version);
}
for (auto & e : json["elements"]) {
ProfileElement element;
for (auto & p : e["storePaths"])
element.storePaths.insert(state.store->parseStorePath((std::string) p));
element.active = e["active"];
if (e.value("uri", "") != "") {
if (e.value(sUrl,"") != "") {
element.source = ProfileElementSource{
parseFlakeRef(e["originalUri"]),
parseFlakeRef(e["uri"]),
parseFlakeRef(e[sOriginalUrl]),
parseFlakeRef(e[sUrl]),
e["attrPath"]
};
}
@ -143,14 +154,14 @@ struct ProfileManifest
obj["storePaths"] = paths;
obj["active"] = element.active;
if (element.source) {
obj["originalUri"] = element.source->originalRef.to_string();
obj["uri"] = element.source->resolvedRef.to_string();
obj["originalUrl"] = element.source->originalRef.to_string();
obj["url"] = element.source->resolvedRef.to_string();
obj["attrPath"] = element.source->attrPath;
}
array.push_back(obj);
}
nlohmann::json json;
json["version"] = 1;
json["version"] = 2;
json["elements"] = array;
return json.dump();
}
@ -235,6 +246,16 @@ struct ProfileManifest
}
};
static std::map<Installable *, BuiltPaths>
builtPathsPerInstallable(
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
{
std::map<Installable *, BuiltPaths> res;
for (auto & [installable, builtPath] : builtPaths)
res[installable.get()].push_back(builtPath);
return res;
}
struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
{
std::string description() override
@ -253,7 +274,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
{
ProfileManifest manifest(*getEvalState(), *profile);
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
auto builtPaths = builtPathsPerInstallable(
Installable::build2(
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
for (auto & installable : installables) {
ProfileElement element;
@ -268,7 +291,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
};
}
element.updateStorePaths(getEvalStore(), store, *installable);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
manifest.elements.push_back(std::move(element));
}
@ -456,12 +479,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
warn ("Use 'nix profile list' to see the current profile.");
}
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
auto builtPaths = builtPathsPerInstallable(
Installable::build2(
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)];
element.updateStorePaths(getEvalStore(), store, *installable);
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
}
updateProfile(manifest.build(store));

6
src/nix/profile.md
View file

@ -70,7 +70,7 @@ are installed in this version of the profile. It looks like this:
{
"active": true,
"attrPath": "legacyPackages.x86_64-linux.zoom-us",
"originalUri": "flake:nixpkgs",
"originalUrl": "flake:nixpkgs",
"storePaths": [
"/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
],
@ -84,11 +84,11 @@ are installed in this version of the profile. It looks like this:
Each object in the array `elements` denotes an installed package and
has the following fields:
* `originalUri`: The [flake reference](./nix3-flake.md) specified by
* `originalUrl`: The [flake reference](./nix3-flake.md) specified by
the user at the time of installation (e.g. `nixpkgs`). This is also
the flake reference that will be used by `nix profile upgrade`.
* `uri`: The immutable flake reference to which `originalUri`
* `uri`: The immutable flake reference to which `originalUrl`
resolved.
* `attrPath`: The flake output attribute that provided this

1
src/nix/repl.cc
View file

@ -396,6 +396,7 @@ StorePath NixRepl::getDerivationPath(Value & v) {
bool NixRepl::processLine(std::string line)
{
line = trim(line);
if (line == "") return true;
_isInterrupted = false;

8
src/nix/run.cc
View file

@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded program
(which "nix" is), so we exec() a single-threaded helper program
(chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>();
auto store2 = store.dynamic_pointer_cast<LocalFSStore>();
if (store2 && store->storeDir != store2->getRealStoreDir()) {
if (!store2)
throw Error("store '%s' is not a local store so it does not support command execution", store->getUri());
if (store->storeDir != store2->getRealStoreDir()) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
for (auto & arg : args) helperArgs.push_back(arg);
@ -179,6 +182,7 @@ struct CmdRun : InstallableCommand
{
auto state = getEvalState();
lockFlags.applyNixConfig = true;
auto app = installable->toApp(*state).resolve(getEvalStore(), store);
Strings allArgs{app.program};

4
src/nix/show-derivation.cc
View file

@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
outputObj.attr("impure", true);
},
}, output.raw());
}
}

2
tests/binary-cache.sh
View file

@ -1,6 +1,6 @@
source common.sh
needLocalStore "“--no-require-sigs” can’t be used with the daemon"
needLocalStore "'--no-require-sigs' can’t be used with the daemon"
# We can produce drvs directly into the binary cache
clearStore

14
tests/build.sh
View file

@ -1,15 +1,27 @@
source common.sh
expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
clearStore
# Make sure that 'nix build' only returns the outputs we asked for.
nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 1) and
(.outputs.first | match(".*multiple-outputs-a-first")))
'
nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys | length == 2) and
(.outputs.first | match(".*multiple-outputs-a-first")) and
(.outputs.second | match(".*multiple-outputs-a-second")))
and (.[1] |
(.drvPath | match(".*multiple-outputs-b.drv")) and
(.outputs | keys | length == 1) and
(.outputs.out | match(".*multiple-outputs-b")))
'
testNormalization () {
clearStore
outPath=$(nix-build ./simple.nix --no-out-link)

3
tests/ca/content-addressed.nix
View file

@ -64,8 +64,7 @@ rec {
dependentFixedOutput = mkDerivation {
name = "dependent-fixed-output";
outputHashMode = "recursive";
outputHashAlgo = "sha256";
outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=";
outputHash = "sha512-7aJcmSuEuYP5tGKcmGY8bRr/lrCjJlOxP2mIUjO/vMQeg6gx/65IbzRWES8EKiPDOs9z+wF30lEfcwxM/cT4pw==";
buildCommand = ''
cat ${dependentCA}/dep
echo foo > $out

70
tests/fetchClosure.sh Normal file
View file

@ -0,0 +1,70 @@
source common.sh
enableFeatures "fetch-closure"
needLocalStore "'--no-require-sigs' can’t be used with the daemon"
clearStore
clearCacheCache
# Initialize binary cache.
nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out)
caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]')
nix copy --to file://$cacheDir $nonCaPath
# Test basic fetchClosure rewriting from non-CA to CA.
clearStore
[ ! -e $nonCaPath ]
[ ! -e $caPath ]
[[ $(nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
toPath = $caPath;
}
") = $caPath ]]
[ ! -e $nonCaPath ]
[ -e $caPath ]
# In impure mode, we can use non-CA paths.
[[ $(nix eval --raw --no-require-sigs --impure --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
}
") = $nonCaPath ]]
[ -e $nonCaPath ]
# 'toPath' set to empty string should fail but print the expected path.
nix eval -v --json --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
toPath = \"\";
}
" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath"
# If fromPath is CA, then toPath isn't needed.
nix copy --to file://$cacheDir $caPath
[[ $(nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $caPath;
}
") = $caPath ]]
# Check that URL query parameters aren't allowed.
clearStore
narCache=$TEST_ROOT/nar-cache
rm -rf $narCache
(! nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir?local-nar-cache=$narCache\";
fromPath = $caPath;
}
")
(! [ -e $narCache ])

4
tests/fetchGit.sh
View file

@ -7,7 +7,9 @@ fi
clearStore
repo=$TEST_ROOT/git
# Intentionally not in a canonical form
# See https://github.com/NixOS/nix/issues/6195
repo=$TEST_ROOT/./git
export _NIX_FORCE_HTTP=1

4
tests/fetchPath.sh
View file

@ -1,6 +1,6 @@
source common.sh
touch foo -t 202211111111
touch $TEST_ROOT/foo -t 202211111111
# We only check whether 2022-11-1* **:**:** is the last modified date since
# `lastModified` is transformed into UTC in `builtins.fetchTarball`.
[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]]
[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]]

3
tests/flakes.sh
View file

@ -376,6 +376,9 @@ cat > $templatesDir/flake.nix <<EOF
trivial = {
path = ./trivial;
description = "A trivial flake";
welcomeText = ''
Welcome to my trivial flake
'';
};
default = trivial;
};

63
tests/impure-derivations.nix Normal file
View file

@ -0,0 +1,63 @@
with import ./config.nix;
rec {
impure = mkDerivation {
name = "impure";
outputs = [ "out" "stuff" ];
buildCommand =
''
echo impure
x=$(< $TEST_ROOT/counter)
mkdir $out $stuff
echo $x > $out/n
ln -s $out/n $stuff/bla
printf $((x + 1)) > $TEST_ROOT/counter
'';
__impure = true;
impureEnvVars = [ "TEST_ROOT" ];
};
impureOnImpure = mkDerivation {
name = "impure-on-impure";
buildCommand =
''
echo impure-on-impure
x=$(< ${impure}/n)
mkdir $out
printf X$x > $out/n
ln -s ${impure.stuff} $out/symlink
ln -s $out $out/self
'';
__impure = true;
};
# This is not allowed.
inputAddressed = mkDerivation {
name = "input-addressed";
buildCommand =
''
cat ${impure} > $out
'';
};
contentAddressed = mkDerivation {
name = "content-addressed";
buildCommand =
''
echo content-addressed
x=$(< ${impureOnImpure}/n)
printf ''${x:0:1} > $out
'';
outputHashMode = "recursive";
outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ=";
};
inputAddressedAfterCA = mkDerivation {
name = "input-addressed-after-ca";
buildCommand =
''
cat ${contentAddressed} > $out
'';
};
}

57
tests/impure-derivations.sh Normal file
View file

@ -0,0 +1,57 @@
source common.sh
requireDaemonNewerThan "2.8pre20220311"
enableFeatures "ca-derivations ca-references impure-derivations"
restartDaemon
set -o pipefail
clearStore
# Basic test of impure derivations: building one a second time should not use the previous result.
printf 0 > $TEST_ROOT/counter
json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
path1=$(echo $json | jq -r .[].outputs.out)
path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
[[ $(< $path1/n) = 0 ]]
[[ $(< $path1_stuff/bla) = 0 ]]
[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
[[ $(< $path2/n) = 1 ]]
# Test impure derivations that depend on impure derivations.
path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path3/n) = X2 ]]
path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path4/n) = X3 ]]
# Test that (self-)references work.
[[ $(< $path4/symlink/bla) = 3 ]]
[[ $(< $path4/self/n) = X3 ]]
# Input-addressed derivations cannot depend on impure derivations directly.
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
# Fixed-output derivations *can* depend on impure derivations.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# And they should not be rebuilt.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations.
path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out)
[[ $(< $path6) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]

4
tests/local.mk
View file

@ -96,7 +96,9 @@ nix_tests = \
describe-stores.sh \
nix-profile.sh \
suggestions.sh \
store-ping.sh
store-ping.sh \
fetchClosure.sh \
impure-derivations.sh
ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh

11
tests/logging.sh
View file

@ -13,3 +13,14 @@ rm -rf $NIX_LOG_DIR
(! nix-store -l $path)
nix-build dependencies.nix --no-out-link --compress-build-log
[ "$(nix-store -l $path)" = FOO ]
# test whether empty logs work fine with `nix log`.
builder="$(mktemp)"
echo -e "#!/bin/sh\nmkdir \$out" > "$builder"
outp="$(nix-build -E \
'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \
--out-link "$(mktemp -d)/result")"
test -d "$outp"
nix log "$outp"

2
tests/sourcehut-flakes.nix
View file

@ -59,7 +59,7 @@ let
echo 'ref: refs/heads/master' > $out/HEAD
mkdir -p $out/info
echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs
echo -e '${nixpkgs.rev}\trefs/heads/master' > $out/info/refs
'';
in

3
tests/user-envs.nix
View file

@ -8,6 +8,8 @@ assert foo == "foo";
let
platforms = let x = "foobar"; in [ x x ];
makeDrv = name: progName: (mkDerivation {
name = assert progName != "fail"; name;
inherit progName system;
@ -15,6 +17,7 @@ let
} // {
meta = {
description = "A silly test package with some \${escaped anti-quotation} in it";
inherit platforms;
};
});