mirror of
https://github.com/NixOS/nix.git
synced 2025-11-15 15:02:42 +01:00
Merge branch 'structured-attrs-shell' of https://github.com/Ma27/nix
This commit is contained in:
Eelco Dolstra
commit
e06c272c12
13 changed files with 270 additions and 148 deletions
|
|
@ -143,7 +143,6 @@ void DerivationGoal::work()
|
|||
(this->*state)();
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
|
||||
{
|
||||
/* If we already want all outputs, there is nothing to do. */
|
||||
|
|
@ -1074,42 +1073,6 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
|
||||
StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
|
||||
{
|
||||
StorePathSet paths;
|
||||
|
||||
for (auto & storePath : storePaths) {
|
||||
if (!inputPaths.count(storePath))
|
||||
throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
|
||||
|
||||
worker.store.computeFSClosure({storePath}, paths);
|
||||
}
|
||||
|
||||
/* If there are derivations in the graph, then include their
|
||||
outputs as well. This is useful if you want to do things
|
||||
like passing all build-time dependencies of some path to a
|
||||
derivation that builds a NixOS DVD image. */
|
||||
auto paths2 = paths;
|
||||
|
||||
for (auto & j : paths2) {
|
||||
if (j.isDerivation()) {
|
||||
Derivation drv = worker.store.derivationFromPath(j);
|
||||
for (auto & k : drv.outputsAndOptPaths(worker.store)) {
|
||||
if (!k.second.second)
|
||||
/* FIXME: I am confused why we are calling
|
||||
`computeFSClosure` on the output path, rather than
|
||||
derivation itself. That doesn't seem right to me, so I
|
||||
won't try to implemented this for CA derivations. */
|
||||
throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
|
||||
worker.store.computeFSClosure(*k.second.second, paths);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
|
|
|
|||
|
|
@ -518,7 +518,7 @@ void LocalDerivationGoal::startBuilder()
|
|||
/* Write closure info to <fileName>. */
|
||||
writeFile(tmpDir + "/" + fileName,
|
||||
worker.store.makeValidityRegistration(
|
||||
exportReferences({storePath}), false, false));
|
||||
worker.store.exportReferences({storePath}, inputPaths), false, false));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1084,113 +1084,28 @@ void LocalDerivationGoal::initEnv()
|
|||
}
|
||||
|
||||
|
||||
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
|
||||
|
||||
|
||||
void LocalDerivationGoal::writeStructuredAttrs()
|
||||
{
|
||||
auto structuredAttrs = parsedDrv->getStructuredAttrs();
|
||||
if (!structuredAttrs) return;
|
||||
if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) {
|
||||
auto json = structAttrsJson.value();
|
||||
nlohmann::json rewritten;
|
||||
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
|
||||
/* The placeholder must have a rewrite, so we use it to cover both the
|
||||
cases where we know or don't know the output path ahead of time. */
|
||||
rewritten[i] = rewriteStrings(v, inputRewrites);
|
||||
}
|
||||
|
||||
auto json = *structuredAttrs;
|
||||
json["outputs"] = rewritten;
|
||||
|
||||
/* Add an "outputs" object containing the output paths. */
|
||||
nlohmann::json outputs;
|
||||
for (auto & i : drv->outputs) {
|
||||
/* The placeholder must have a rewrite, so we use it to cover both the
|
||||
cases where we know or don't know the output path ahead of time. */
|
||||
outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
|
||||
auto jsonSh = writeStructuredAttrsShell(json);
|
||||
|
||||
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
|
||||
chownToBuilder(tmpDir + "/.attrs.sh");
|
||||
env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
|
||||
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
|
||||
chownToBuilder(tmpDir + "/.attrs.json");
|
||||
env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
|
||||
}
|
||||
json["outputs"] = outputs;
|
||||
|
||||
/* Handle exportReferencesGraph. */
|
||||
auto e = json.find("exportReferencesGraph");
|
||||
if (e != json.end() && e->is_object()) {
|
||||
for (auto i = e->begin(); i != e->end(); ++i) {
|
||||
std::ostringstream str;
|
||||
{
|
||||
JSONPlaceholder jsonRoot(str, true);
|
||||
StorePathSet storePaths;
|
||||
for (auto & p : *i)
|
||||
storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
|
||||
worker.store.pathInfoToJSON(jsonRoot,
|
||||
exportReferences(storePaths), false, true);
|
||||
}
|
||||
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
|
||||
}
|
||||
}
|
||||
|
||||
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
|
||||
chownToBuilder(tmpDir + "/.attrs.json");
|
||||
|
||||
/* As a convenience to bash scripts, write a shell file that
|
||||
maps all attributes that are representable in bash -
|
||||
namely, strings, integers, nulls, Booleans, and arrays and
|
||||
objects consisting entirely of those values. (So nested
|
||||
arrays or objects are not supported.) */
|
||||
|
||||
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
|
||||
if (value.is_string())
|
||||
return shellEscape(value);
|
||||
|
||||
if (value.is_number()) {
|
||||
auto f = value.get<float>();
|
||||
if (std::ceil(f) == f)
|
||||
return std::to_string(value.get<int>());
|
||||
}
|
||||
|
||||
if (value.is_null())
|
||||
return std::string("''");
|
||||
|
||||
if (value.is_boolean())
|
||||
return value.get<bool>() ? std::string("1") : std::string("");
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
std::string jsonSh;
|
||||
|
||||
for (auto i = json.begin(); i != json.end(); ++i) {
|
||||
|
||||
if (!std::regex_match(i.key(), shVarName)) continue;
|
||||
|
||||
auto & value = i.value();
|
||||
|
||||
auto s = handleSimpleType(value);
|
||||
if (s)
|
||||
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
|
||||
|
||||
else if (value.is_array()) {
|
||||
std::string s2;
|
||||
bool good = true;
|
||||
|
||||
for (auto i = value.begin(); i != value.end(); ++i) {
|
||||
auto s3 = handleSimpleType(i.value());
|
||||
if (!s3) { good = false; break; }
|
||||
s2 += *s3; s2 += ' ';
|
||||
}
|
||||
|
||||
if (good)
|
||||
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
|
||||
}
|
||||
|
||||
else if (value.is_object()) {
|
||||
std::string s2;
|
||||
bool good = true;
|
||||
|
||||
for (auto i = value.begin(); i != value.end(); ++i) {
|
||||
auto s3 = handleSimpleType(i.value());
|
||||
if (!s3) { good = false; break; }
|
||||
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
|
||||
}
|
||||
|
||||
if (good)
|
||||
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
|
||||
}
|
||||
}
|
||||
|
||||
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
|
||||
chownToBuilder(tmpDir + "/.attrs.sh");
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#include "parsed-derivations.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <regex>
|
||||
#include "json.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -123,4 +125,109 @@ bool ParsedDerivation::substitutesAllowed() const
|
|||
return getBoolAttr("allowSubstitutes", true);
|
||||
}
|
||||
|
||||
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
|
||||
std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
|
||||
{
|
||||
auto structuredAttrs = getStructuredAttrs();
|
||||
if (!structuredAttrs) return std::nullopt;
|
||||
|
||||
auto json = *structuredAttrs;
|
||||
|
||||
/* Add an "outputs" object containing the output paths. */
|
||||
nlohmann::json outputs;
|
||||
for (auto & i : drv.outputs) {
|
||||
outputs[i.first] = hashPlaceholder(i.first);
|
||||
}
|
||||
json["outputs"] = outputs;
|
||||
|
||||
/* Handle exportReferencesGraph. */
|
||||
auto e = json.find("exportReferencesGraph");
|
||||
if (e != json.end() && e->is_object()) {
|
||||
for (auto i = e->begin(); i != e->end(); ++i) {
|
||||
std::ostringstream str;
|
||||
{
|
||||
JSONPlaceholder jsonRoot(str, true);
|
||||
StorePathSet storePaths;
|
||||
for (auto & p : *i)
|
||||
storePaths.insert(store.parseStorePath(p.get<std::string>()));
|
||||
store.pathInfoToJSON(jsonRoot,
|
||||
store.exportReferences(storePaths, inputPaths), false, true);
|
||||
}
|
||||
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
|
||||
}
|
||||
}
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
/* As a convenience to bash scripts, write a shell file that
|
||||
maps all attributes that are representable in bash -
|
||||
namely, strings, integers, nulls, Booleans, and arrays and
|
||||
objects consisting entirely of those values. (So nested
|
||||
arrays or objects are not supported.) */
|
||||
std::string writeStructuredAttrsShell(nlohmann::json & json)
|
||||
{
|
||||
|
||||
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
|
||||
if (value.is_string())
|
||||
return shellEscape(value);
|
||||
|
||||
if (value.is_number()) {
|
||||
auto f = value.get<float>();
|
||||
if (std::ceil(f) == f)
|
||||
return std::to_string(value.get<int>());
|
||||
}
|
||||
|
||||
if (value.is_null())
|
||||
return std::string("''");
|
||||
|
||||
if (value.is_boolean())
|
||||
return value.get<bool>() ? std::string("1") : std::string("");
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
std::string jsonSh;
|
||||
|
||||
for (auto i = json.begin(); i != json.end(); ++i) {
|
||||
|
||||
if (!std::regex_match(i.key(), shVarName)) continue;
|
||||
|
||||
auto & value = i.value();
|
||||
|
||||
auto s = handleSimpleType(value);
|
||||
if (s)
|
||||
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
|
||||
|
||||
else if (value.is_array()) {
|
||||
std::string s2;
|
||||
bool good = true;
|
||||
|
||||
for (auto i = value.begin(); i != value.end(); ++i) {
|
||||
auto s3 = handleSimpleType(i.value());
|
||||
if (!s3) { good = false; break; }
|
||||
s2 += *s3; s2 += ' ';
|
||||
}
|
||||
|
||||
if (good)
|
||||
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
|
||||
}
|
||||
|
||||
else if (value.is_object()) {
|
||||
std::string s2;
|
||||
bool good = true;
|
||||
|
||||
for (auto i = value.begin(); i != value.end(); ++i) {
|
||||
auto s3 = handleSimpleType(i.value());
|
||||
if (!s3) { good = false; break; }
|
||||
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
|
||||
}
|
||||
|
||||
if (good)
|
||||
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
|
||||
}
|
||||
}
|
||||
|
||||
return jsonSh;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ public:
|
|||
bool willBuildLocally(Store & localStore) const;
|
||||
|
||||
bool substitutesAllowed() const;
|
||||
|
||||
std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
|
||||
};
|
||||
|
||||
std::string writeStructuredAttrsShell(nlohmann::json & json);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -634,6 +634,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
|
|||
}
|
||||
|
||||
|
||||
StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths)
|
||||
{
|
||||
StorePathSet paths;
|
||||
|
||||
for (auto & storePath : storePaths) {
|
||||
if (!inputPaths.count(storePath))
|
||||
throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath));
|
||||
|
||||
computeFSClosure({storePath}, paths);
|
||||
}
|
||||
|
||||
/* If there are derivations in the graph, then include their
|
||||
outputs as well. This is useful if you want to do things
|
||||
like passing all build-time dependencies of some path to a
|
||||
derivation that builds a NixOS DVD image. */
|
||||
auto paths2 = paths;
|
||||
|
||||
for (auto & j : paths2) {
|
||||
if (j.isDerivation()) {
|
||||
Derivation drv = derivationFromPath(j);
|
||||
for (auto & k : drv.outputsAndOptPaths(*this)) {
|
||||
if (!k.second.second)
|
||||
/* FIXME: I am confused why we are calling
|
||||
`computeFSClosure` on the output path, rather than
|
||||
derivation itself. That doesn't seem right to me, so I
|
||||
won't try to implemented this for CA derivations. */
|
||||
throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
|
||||
computeFSClosure(*k.second.second, paths);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
|
||||
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
|
||||
bool includeImpureInfo, bool showClosureSize,
|
||||
Base hashBase,
|
||||
|
|
|
|||
|
|
@ -697,6 +697,11 @@ public:
|
|||
|
||||
const Stats & getStats();
|
||||
|
||||
/* Computes the full closure of of a set of store-paths for e.g.
|
||||
derivations that need this information for `exportReferencesGraph`.
|
||||
*/
|
||||
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
|
||||
|
||||
/* Return the build log of the specified store path, if available,
|
||||
or null otherwise. */
|
||||
virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue