nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 20:16:03 +01:00
Code Activity

libexpr: Structural sharing of attrsets

Browse source 
This changes the implementation of Bindings to allow
for a more space-efficient implementation of attribute
set merges. This is accomplished by "layering" over the "base" Bindings.
The top "layer" is naturally the right-hand-side of the update operator //.

Such an implementation leads to significantly better memory usage on
something like nixpkgs:

nix-env --query --available --out-path --file ../nixpkgs --eval-system x86_64-linux > /dev/null

Comparison against 2b0fd88324 for x86_64-linux on nixpkgs f06c7c3b6f5074dbffcf02542fb86af3a5526afa:

| metric                 | mean_before     | mean_after      | mean_diff       | mean_%_change | p_value | t_stat  |
| -                      | -               | -               | -               | -             | -       | -       |
| cpuTime                | 21.1520         | 21.3414         | 0.1894          | 0.7784        | 0.3190  | 1.0219  |
| envs.bytes             | 461451951.6190  | 461451951.6190  | -               | -             | -       | -       |
| envs.elements          | 34344544.8571   | 34344544.8571   | -               | -             | -       | -       |
| envs.number            | 23336949.0952   | 23336949.0952   | -               | -             | -       | -       |
| gc.cycles              | 7.5238          | 7.2857          | -0.2381         | -4.6825       | 0.0565  | -2.0244 |
| gc.heapSize            | 1777848124.9524 | 1252162023.6190 | -525686101.3333 | -29.9472      | 0.0000  | -8.7041 |
| gc.totalBytes          | 3102787383.6190 | 2498431578.6667 | -604355804.9524 | -19.7704      | 0.0000  | -9.3502 |
| list.bytes             | 59928225.9048   | 59928225.9048   | -               | -             | -       | -       |
| list.concats           | 1240028.2857    | 1240028.2857    | -               | -             | -       | -       |
| list.elements          | 7491028.2381    | 7491028.2381    | -               | -             | -       | -       |
| nrAvoided              | 28165342.2381   | 28165342.2381   | -               | -             | -       | -       |
| nrExprs                | 1577412.9524    | 1577412.9524    | -               | -             | -       | -       |
| nrFunctionCalls        | 20970743.4286   | 20970743.4286   | -               | -             | -       | -       |
| nrLookups              | 10867306.0952   | 10867306.0952   | -               | -             | -       | -       |
| nrOpUpdateValuesCopied | 61206062.0000   | 25748169.5238   | -35457892.4762  | -58.8145      | 0.0000  | -8.9189 |
| nrOpUpdates            | 2167097.4286    | 2167097.4286    | -               | -             | -       | -       |
| nrPrimOpCalls          | 12337423.4286   | 12337423.4286   | -               | -             | -       | -       |
| nrThunks               | 29361806.7619   | 29361806.7619   | -               | -             | -       | -       |
| sets.bytes             | 1393822818.6667 | 897587655.2381  | -496235163.4286 | -36.7168      | 0.0000  | -9.1115 |
| sets.elements          | 84504465.3333   | 48270845.9524   | -36233619.3810  | -43.8698      | 0.0000  | -8.9181 |
| sets.number            | 5218921.6667    | 5218921.6667    | -               | -             | -       | -       |
| sizes.Attr             | 16.0000         | 16.0000         | -               | -             | -       | -       |
| sizes.Bindings         | 8.0000          | 24.0000         | 16.0000         | 200.0000      | -       | inf     |
| sizes.Env              | 8.0000          | 8.0000          | -               | -             | -       | -       |
| sizes.Value            | 16.0000         | 16.0000         | -               | -             | -       | -       |
| symbols.bytes          | 1368494.0952    | 1368494.0952    | -               | -             | -       | -       |
| symbols.number         | 109147.1905     | 109147.1905     | -               | -             | -       | -       |
| time.cpu               | 21.1520         | 21.3414         | 0.1894          | 0.7784        | 0.3190  | 1.0219  |
| time.gc                | 1.6011          | 0.8508          | -0.7503         | -37.1507      | 0.0017  | -3.6328 |
| time.gcFraction        | 0.0849          | 0.0399          | -0.0450         | -37.4504      | 0.0035  | -3.3116 |
| values.bytes           | 615968144.7619  | 615968144.7619  | -               | -             | -       | -       |
| values.number          | 38498009.0476   | 38498009.0476   | -               | -             | -       | -       |

Overall this does slow down the evaluator slightly (no more than ~10% in most cases),
but this seems like a very decent tradeoff for shaving off 33% of memory usage.
This commit is contained in:
Sergei Zimmerman 2025-09-15 22:16:17 +03:00
parent 3eb223f4bb
commit 6138bc3de3
No known key found for this signature in database
7 changed files with 424 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

62
src/libexpr/eval.cc
View file

@ -1873,37 +1873,71 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
state.nrOpUpdates++;
if (v1.attrs()->size() == 0) {
const Bindings & bindings1 = *v1.attrs();
if (bindings1.empty()) {
v = v2;
return;
}
if (v2.attrs()->size() == 0) {
const Bindings & bindings2 = *v2.attrs();
if (bindings2.empty()) {
v = v1;
return;
}
auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size());
/* Simple heuristic for determining whether attrs2 should be "layered" on top of
attrs1 instead of copying to a new Bindings. */
bool shouldLayer = [&]() -> bool {
if (bindings1.isLayerListFull())
return false;
if (bindings2.size() > state.settings.bindingsUpdateLayerRhsSizeThreshold)
return false;
return true;
}();
if (shouldLayer) {
auto attrs = state.buildBindings(bindings2.size());
attrs.layerOnTopOf(bindings1);
std::ranges::copy(bindings2, std::back_inserter(attrs));
v.mkAttrs(attrs.alreadySorted());
state.nrOpUpdateValuesCopied += bindings2.size();
return;
}
auto attrs = state.buildBindings(bindings1.size() + bindings2.size());
/* Merge the sets, preferring values from the second set. Make
sure to keep the resulting vector in sorted order. */
auto i = v1.attrs()->begin();
auto j = v2.attrs()->begin();
auto i = bindings1.begin();
auto j = bindings2.begin();
while (i != v1.attrs()->end() && j != v2.attrs()->end()) {
while (i != bindings1.end() && j != bindings2.end()) {
if (i->name == j->name) {
attrs.insert(*j);
++i;
++j;
} else if (i->name < j->name)
attrs.insert(*i++);
else
attrs.insert(*j++);
} else if (i->name < j->name) {
attrs.insert(*i);
++i;
} else {
attrs.insert(*j);
++j;
}
}
while (i != v1.attrs()->end())
attrs.insert(*i++);
while (j != v2.attrs()->end())
attrs.insert(*j++);
while (i != bindings1.end()) {
attrs.insert(*i);
++i;
}
while (j != bindings2.end()) {
attrs.insert(*j);
++j;
}
v.mkAttrs(attrs.alreadySorted());