nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-14 14:32:42 +01:00
Code Activity

Misc Windows fixes

Browse source 
1. Fix build by making the legacy SSH Storey's secret `logFD` setting
   not a setting on Windows. (It doesn't make sense to specify `void *`
   handles by integer cross-proccess, I don't think.)

2. Move some files that don't need to be Unix-only anymore back to their
   original locations.
This commit is contained in:
John Ericson 2024-06-01 16:39:59 -04:00
parent 802b4e403b
commit e0b159549b
11 changed files with 36 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

81
src/libstore/unix/builtins/fetchurl.cc
View file

@ -1,81 +0,0 @@
#include "builtins.hh"
#include "filetransfer.hh"
#include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
namespace nix {
void builtinFetchurl(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs,
const std::string & netrcData)
{
/* Make the host's netrc data available. Too bad curl requires
this to be stored in a file. It would be nice if we could just
pass a pointer to the data. */
if (netrcData != "") {
settings.netrcFile = "netrc";
writeFile(settings.netrcFile, netrcData, 0600);
}
auto out = get(drv.outputs, "out");
if (!out)
throw Error("'builtin:fetchurl' requires an 'out' output");
if (!(drv.type().isFixed() || drv.type().isImpure()))
throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation");
auto storePath = outputs.at("out");
auto mainUrl = drv.env.at("url");
bool unpack = getOr(drv.env, "unpack", "") == "1";
/* Note: have to use a fresh fileTransfer here because we're in
a forked process. */
auto fileTransfer = makeFileTransfer();
auto fetch = [&](const std::string & url) {
auto source = sinkToSource([&](Sink & sink) {
/* No need to do TLS verification, because we check the hash of
the result anyway. */
FileTransferRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
fileTransfer->download(std::move(request), *decompressor);
decompressor->finish();
});
if (unpack)
restorePath(storePath, *source);
else
writeFile(storePath, *source);
auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") {
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError("making '%1%' executable", storePath);
}
};
/* Try the hashed mirrors first. */
auto dof = std::get_if<DerivationOutput::CAFixed>(&out->raw);
if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false));
return;
} catch (Error & e) {
debug(e.what());
}
/* Otherwise try the specified URL. */
fetch(mainUrl);
}
}

33
src/libstore/unix/builtins/unpack-channel.cc
View file

@ -1,33 +0,0 @@
#include "builtins.hh"
#include "tarfile.hh"
namespace nix {
void builtinUnpackChannel(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs)
{
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
auto out = outputs.at("out");
auto channelName = getAttr("channelName");
auto src = getAttr("src");
createDirs(out);
unpackTarfile(src, out);
auto entries = std::filesystem::directory_iterator{out};
auto fileName = entries->path().string();
auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries));
if (fileCount != 1)
throw Error("channel tarball '%s' contains more than one file", src);
std::filesystem::rename(fileName, (out + "/" + channelName));
}
}

292
src/libstore/unix/local-overlay-store.cc
View file

@ -1,292 +0,0 @@
#include "local-overlay-store.hh"
#include "callback.hh"
#include "realisation.hh"
#include "processes.hh"
#include "url.hh"
#include <regex>
namespace nix {
std::string LocalOverlayStoreConfig::doc()
{
return
#include "local-overlay-store.md"
;
}
Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) {
return upperLayer + "/" + path.to_string();
}
LocalOverlayStore::LocalOverlayStore(const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
, LocalOverlayStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
, lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
{
if (checkMount.get()) {
std::smatch match;
std::string mountInfo;
auto mounts = readFile("/proc/self/mounts");
auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))");
// Mount points can be stacked, so there might be multiple matching entries.
// Loop until the last match, which will be the current state of the mount point.
while (std::regex_search(mounts, match, regex)) {
mountInfo = match.str();
mounts = match.suffix();
}
auto checkOption = [&](std::string option, std::string value) {
return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)"));
};
auto expectedLowerDir = lowerStore->realStoreDir.get();
if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) {
debug("expected lowerdir: %s", expectedLowerDir);
debug("expected upperdir: %s", upperLayer);
debug("actual mount: %s", mountInfo);
throw Error("overlay filesystem '%s' mounted incorrectly",
realStoreDir.get());
}
}
}
void LocalOverlayStore::registerDrvOutput(const Realisation & info)
{
// First do queryRealisation on lower layer to populate DB
auto res = lowerStore->queryRealisation(info.id);
if (res)
LocalStore::registerDrvOutput(*res);
LocalStore::registerDrvOutput(info);
}
void LocalOverlayStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
LocalStore::queryPathInfoUncached(path,
{[this, path, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
if (info)
return (*callbackPtr)(std::move(info));
} catch (...) {
return callbackPtr->rethrow();
}
// If we don't have it, check lower store
lowerStore->queryPathInfo(path,
{[path, callbackPtr](std::future<ref<const ValidPathInfo>> fut) {
try {
(*callbackPtr)(fut.get().get_ptr());
} catch (...) {
return callbackPtr->rethrow();
}
}});
}});
}
void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
LocalStore::queryRealisationUncached(drvOutput,
{[this, drvOutput, callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
try {
auto info = fut.get();
if (info)
return (*callbackPtr)(std::move(info));
} catch (...) {
return callbackPtr->rethrow();
}
// If we don't have it, check lower store
lowerStore->queryRealisation(drvOutput,
{[callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
try {
(*callbackPtr)(fut.get());
} catch (...) {
return callbackPtr->rethrow();
}
}});
}});
}
bool LocalOverlayStore::isValidPathUncached(const StorePath & path)
{
auto res = LocalStore::isValidPathUncached(path);
if (res) return res;
res = lowerStore->isValidPath(path);
if (res) {
// Get path info from lower store so upper DB genuinely has it.
auto p = lowerStore->queryPathInfo(path);
// recur on references, syncing entire closure.
for (auto & r : p->references)
if (r != path)
isValidPath(r);
LocalStore::registerValidPath(*p);
}
return res;
}
void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
{
LocalStore::queryReferrers(path, referrers);
lowerStore->queryReferrers(path, referrers);
}
void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers)
{
LocalStore::queryReferrers(path, referrers);
}
StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path)
{
auto res = LocalStore::queryValidDerivers(path);
for (auto p : lowerStore->queryValidDerivers(path))
res.insert(p);
return res;
}
std::optional<StorePath> LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart)
{
auto res = LocalStore::queryPathFromHashPart(hashPart);
if (res)
return res;
else
return lowerStore->queryPathFromHashPart(hashPart);
}
void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos)
{
// First, get any from lower store so we merge
{
StorePathSet notInUpper;
for (auto & [p, _] : infos)
if (!LocalStore::isValidPathUncached(p)) // avoid divergence
notInUpper.insert(p);
auto pathsInLower = lowerStore->queryValidPaths(notInUpper);
ValidPathInfos inLower;
for (auto & p : pathsInLower)
inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p));
LocalStore::registerValidPaths(inLower);
}
// Then do original request
LocalStore::registerValidPaths(infos);
}
void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results)
{
LocalStore::collectGarbage(options, results);
remountIfNecessary();
}
void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
{
auto mergedDir = realStoreDir.get() + "/";
if (path.substr(0, mergedDir.length()) != mergedDir) {
warn("local-overlay: unexpected gc path '%s' ", path);
return;
}
StorePath storePath = {path.substr(mergedDir.length())};
auto upperPath = toUpperPath(storePath);
if (pathExists(upperPath)) {
debug("upper exists: %s", path);
if (lowerStore->isValidPath(storePath)) {
debug("lower exists: %s", storePath.to_string());
// Path also exists in lower store.
// We must delete via upper layer to avoid creating a whiteout.
deletePath(upperPath, bytesFreed);
_remountRequired = true;
} else {
// Path does not exist in lower store.
// So we can delete via overlayfs and not need to remount.
LocalStore::deleteStorePath(path, bytesFreed);
}
}
}
void LocalOverlayStore::optimiseStore()
{
Activity act(*logger, actOptimiseStore);
// Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer
auto paths = queryAllValidPaths();
act.progress(0, paths.size());
uint64_t done = 0;
for (auto & path : paths) {
if (lowerStore->isValidPath(path)) {
uint64_t bytesFreed = 0;
// Deduplicate store path
deleteStorePath(Store::toRealPath(path), bytesFreed);
}
done++;
act.progress(done, paths.size());
}
remountIfNecessary();
}
LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair)
{
StorePathSet done;
auto existsInStoreDir = [&](const StorePath & storePath) {
return pathExists(realStoreDir.get() + "/" + storePath.to_string());
};
bool errors = false;
StorePathSet validPaths;
for (auto & i : queryAllValidPaths())
verifyPath(i, existsInStoreDir, done, validPaths, repair, errors);
return {
.errors = errors,
.validPaths = validPaths,
};
}
void LocalOverlayStore::remountIfNecessary()
{
if (!_remountRequired) return;
if (remountHook.get().empty()) {
warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get());
} else {
runProgram(remountHook, false, {realStoreDir});
}
_remountRequired = false;
}
static RegisterStoreImplementation<LocalOverlayStore, LocalOverlayStoreConfig> regLocalOverlayStore;
}

215
src/libstore/unix/local-overlay-store.hh
View file

@ -1,215 +0,0 @@
#include "local-store.hh"
namespace nix {
/**
* Configuration for `LocalOverlayStore`.
*/
struct LocalOverlayStoreConfig : virtual LocalStoreConfig
{
LocalOverlayStoreConfig(const StringMap & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, LocalStoreConfig(params)
{ }
const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
Must be a store with a store dir on the file system.
Must be used as OverlayFS lower layer for this store's store dir.
)"};
const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)"};
Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
R"(
Check that the overlay filesystem is correctly mounted.
Nix does not manage the overlayfs mount point itself, but the correct
functioning of the overlay store does depend on this mount point being set up
correctly. Rather than just assume this is the case, check that the lowerdir
and upperdir options are what we expect them to be. This check is on by
default, but can be disabled if needed.
)"};
const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
R"(
Script or other executable to run when overlay filesystem needs remounting.
This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
is the only way to perform the deletion without creating a "whiteout".
However this causes the OverlayFS kernel data structures to get out-of-sync,
and can lead to 'stale file handle' errors; remounting solves the problem.
The store directory is passed as an argument to the invoked executable.
)"};
const std::string name() override { return "Experimental Local Overlay Store"; }
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return ExperimentalFeature::LocalOverlayStore;
}
std::string doc() override;
protected:
/**
* @return The host OS path corresponding to the store path for the
* upper layer.
*
* @note The there is no guarantee a store object is actually stored
* at that file path. It might be stored in the lower layer instead,
* or it might not be part of this store at all.
*/
Path toUpperPath(const StorePath & path);
};
/**
* Variation of local store using OverlayFS for the store directory.
*
* Documentation on overridden methods states how they differ from their
* `LocalStore` counterparts.
*/
class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore
{
/**
* The store beneath us.
*
* Our store dir should be an overlay fs where the lower layer
* is that store's store dir, and the upper layer is some
* scratch storage just for us.
*/
ref<LocalFSStore> lowerStore;
public:
LocalOverlayStore(const Params & params);
LocalOverlayStore(std::string_view scheme, PathView path, const Params & params)
: LocalOverlayStore(params)
{
if (!path.empty())
throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params");
}
static std::set<std::string> uriSchemes()
{
return { "local-overlay" };
}
std::string getUri() override
{
return "local-overlay://";
}
private:
/**
* First copy up any lower store realisation with the same key, so we
* merge rather than mask it.
*/
void registerDrvOutput(const Realisation & info) override;
/**
* Check lower store if upper DB does not have.
*/
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
/**
* Check lower store if upper DB does not have.
*
* In addition, copy up metadata for lower store objects (and their
* closure). (I.e. Optimistically cache in the upper DB.)
*/
bool isValidPathUncached(const StorePath & path) override;
/**
* Check the lower store and upper DB.
*/
void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
/**
* Check the lower store and upper DB.
*/
StorePathSet queryValidDerivers(const StorePath & path) override;
/**
* Check lower store if upper DB does not have.
*/
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
/**
* First copy up any lower store realisation with the same key, so we
* merge rather than mask it.
*/
void registerValidPaths(const ValidPathInfos & infos) override;
/**
* Check lower store if upper DB does not have.
*/
void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
/**
* Call `remountIfNecessary` after collecting garbage normally.
*/
void collectGarbage(const GCOptions & options, GCResults & results) override;
/**
* Check which layers the store object exists in to try to avoid
* needing to remount.
*/
void deleteStorePath(const Path & path, uint64_t & bytesFreed) override;
/**
* Deduplicate by removing store objects from the upper layer that
* are now in the lower layer.
*
* Operations on a layered store will not cause duplications, but addition of
* new store objects to the lower layer can instill induce them
* (there is no way to prevent that). This cleans up those
* duplications.
*
* @note We do not yet optomise the upper layer in the normal way
* (hardlink) yet. We would like to, but it requires more
* refactoring of existing code to support this sustainably.
*/
void optimiseStore() override;
/**
* Check all paths registered in the upper DB.
*
* Note that this includes store objects that reside in either overlayfs layer;
* just enumerating the contents of the upper layer would skip them.
*
* We don't verify the contents of both layers on the assumption that the lower layer is far bigger,
* and also the observation that anything not in the upper db the overlayfs doesn't yet care about.
*/
VerificationResult verifyAllValidPaths(RepairFlag repair) override;
/**
* Deletion only effects the upper layer, so we ignore lower-layer referrers.
*/
void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override;
/**
* Call the `remountHook` if we have done something such that the
* OverlayFS needed to be remounted. See that hook's user-facing
* documentation for further details.
*/
void remountIfNecessary();
/**
* State for `remountIfNecessary`
*/
std::atomic_bool _remountRequired = false;
};
}

131
src/libstore/unix/local-overlay-store.md
View file

@ -1,131 +0,0 @@
R"(
**Store URL format**: `local-overlay`
This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short).
Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store].
("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.)
To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly).
### Conceptual parts of a local overlay store
*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference.
For more "practical" instructions, see the worked-out example in the next subsection.*
The parts of a local overlay store are as follows:
- **Lower store**:
> Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting.
This is any store implementation that includes a store directory as part of the native operating system filesystem.
For example, this could be a [local store], [local daemon store], or even another local overlay store.
The local overlay store never tries to modify the lower store in any way.
Something else could modify the lower store, but there are restrictions on this
Nix itself requires that this store only grow, and not change in other ways.
For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects.
(In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.)
The lower store must not change while it is mounted as part of an overlay store.
To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`).
- **Lower store directory**:
> Specified with `lower-store.real` setting.
This is the directory used/exposed by the lower store.
As specified above, Nix requires the local store can only grow not change in other ways.
Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all.
That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change.
- **Lower metadata source**:
> Not directly specified.
> A consequence of the `lower-store` setting, depending on the type of lower store chosen.
This is abstract, just some way to read the metadata of lower store [store objects][store object].
For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]).
This need not be writable.
As stated above a local overlay store never tries to modify its lower store.
The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are.
- **Upper almost-store**:
> Not directly specified.
> Instead the constituent parts are independently specified as described below.
This is almost but not quite just a [local store].
That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted.
But combined with everything else as part of an overlay local store, it is valid.
- **Upper layer directory**:
> Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting.
This contains additional [store objects][store object]
(or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with).
- **Upper store directory**:
> Specified with the [`real`](#store-experimental-local-overlay-store-real) setting.
> This the same as the base local store setting, and can also be indirectly specified with the [`root`](#store-experimental-local-overlay-store-root) setting.
This contains all the store objects from each of the two directories.
The lower store directory and upper layer directory are combined via OverlayFS to create this directory.
Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first.
Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings.
- **Upper SQLite database**:
> Not directly specified.
> The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting.
> It is is always `${state}/db`.
This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta.
The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer.
(This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.)
[file system object]: @docroot@/store/file-system-object.md
[store object]: @docroot@/store/store-object.md
### Example filesystem layout
Here is a worked out example of usage, following the concepts in the previous section.
Say we have the following paths:
- `/mnt/example/merged-store/nix/store`
- `/mnt/example/store-a/nix/store`
- `/mnt/example/store-b`
Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`:
```
local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b
```
The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`.
Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly:
```shell
mount -t overlay overlay \
-o lowerdir="/mnt/example/store-a/nix/store" \
-o upperdir="/mnt/example/store-b" \
-o workdir="/mnt/example/workdir" \
"/mnt/example/merged-store/nix/store"
```
Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`.
By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not.
You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to.
)"