nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-09 03:56:01 +01:00
Code Activity

Merge pull request #13980 from obsidiansystems/drv-json-issue-13570

Browse source 
Make the JSON format for derivation use basename store paths
This commit is contained in:
Robert Hensing 2025-09-17 23:31:41 +02:00 • committed by GitHub
commit b4fcb64276
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 275 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

17
doc/manual/rl-next/derivation-json.md Normal file
View file

@ -0,0 +1,17 @@
---
synopsis: Derivation JSON format now uses store path basenames (no store dir) only
prs: [13980]
issues: [13570]
---
Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde
in Rust, and Aeson in Haskell), has show that the use of the store dir
in JSON formats is an impediment to systematic JSON formats, because it
requires the serializer/deserializer to take an extra paramater (the
store dir).
We ultimately want to rectify this issue with all (non-stable, able to
be changed) JSON formats. To start with, we are changing the JSON format
for derivations because the `nix derivation` commands are --- in
addition to being formally unstable --- less widely used than other
unstable commands.

29
doc/manual/source/protocols/json/derivation.md
View file

@ -14,6 +14,21 @@ is a JSON object with the following fields:
The name of the derivation. The name of the derivation.
This is used when calculating the store paths of the derivation's outputs. This is used when calculating the store paths of the derivation's outputs.
* `version`:
Must be `3`.
This is a guard that allows us to continue evolving this format.
The choice of `3` is fairly arbitrary, but corresponds to this informal version:
- Version 0: A-Term format
- Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
- Version 2: Separate `method` and `hashAlgo` fields in output specs
- Verison 3: Drop store dir from store paths, just include base name.
Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
* `outputs`: * `outputs`:
Information about the output paths of the derivation. Information about the output paths of the derivation.
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
@ -52,7 +67,6 @@ is a JSON object with the following fields:
> ```json > ```json
> "outputs": { > "outputs": {
> "out": { > "out": {
> "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
> "method": "nar", > "method": "nar",
> "hashAlgo": "sha256", > "hashAlgo": "sha256",
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
@ -63,6 +77,15 @@ is a JSON object with the following fields:
* `inputSrcs`: * `inputSrcs`:
A list of store paths on which this derivation depends. A list of store paths on which this derivation depends.
> **Example**
>
> ```json
> "inputSrcs": [
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
> ]
> ```
* `inputDrvs`: * `inputDrvs`:
A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
@ -70,8 +93,8 @@ is a JSON object with the following fields:
> >
> ```json > ```json
> "inputDrvs": { > "inputDrvs": {
> "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
> "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
> } > }
> ``` > ```

2
src/libstore-c/nix_api_store.cc
View file

@ -181,7 +181,7 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
auto drv = nix::Derivation::fromJSON(*store->ptr, nlohmann::json::parse(json)); auto drv = static_cast<nix::Derivation>(nlohmann::json::parse(json));
auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true); auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true);

3
src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json
View file

@ -21,5 +21,6 @@
"method": "nar" "method": "nar"
} }
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

3
src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json
View file

@ -32,5 +32,6 @@
], ],
"system": "my-system" "system": "my-system"
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

9
src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json
View file

@ -10,14 +10,14 @@
"out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
"out" "out"
] ]
}, },
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
@ -26,7 +26,7 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
], ],
"name": "advanced-attributes-structured-attrs", "name": "advanced-attributes-structured-attrs",
"outputs": { "outputs": {
@ -100,5 +100,6 @@
], ],
"system": "my-system" "system": "my-system"
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

9
src/libstore-tests/data/derivation/ca/advanced-attributes.json
View file

@ -26,14 +26,14 @@
"system": "my-system" "system": "my-system"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
"out" "out"
] ]
}, },
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
@ -42,7 +42,7 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
], ],
"name": "advanced-attributes", "name": "advanced-attributes",
"outputs": { "outputs": {
@ -51,5 +51,6 @@
"method": "nar" "method": "nar"
} }
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

3
src/libstore-tests/data/derivation/ca/self-contained.json
View file

@ -19,5 +19,6 @@
"method": "nar" "method": "nar"
} }
}, },
"system": "x86_64-linux" "system": "x86_64-linux",
"version": 3
} }

7
src/libstore-tests/data/derivation/dynDerivationDeps.json
View file

@ -8,7 +8,7 @@
"BIG_BAD": "WOLF" "BIG_BAD": "WOLF"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": { "dynamicOutputs": {
"cat": { "cat": {
"dynamicOutputs": {}, "dynamicOutputs": {},
@ -30,9 +30,10 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
], ],
"name": "dyn-dep-derivation", "name": "dyn-dep-derivation",
"outputs": {}, "outputs": {},
"system": "wasm-sel4" "system": "wasm-sel4",
"version": 3
} }

5
src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json
View file

@ -15,8 +15,9 @@
"name": "advanced-attributes-defaults", "name": "advanced-attributes-defaults",
"outputs": { "outputs": {
"out": { "out": {
"path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" "path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"
} }
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

7
src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json
View file

@ -13,10 +13,10 @@
"name": "advanced-attributes-structured-attrs-defaults", "name": "advanced-attributes-structured-attrs-defaults",
"outputs": { "outputs": {
"dev": { "dev": {
"path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" "path": "8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"
}, },
"out": { "out": {
"path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" "path": "f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
} }
}, },
"structuredAttrs": { "structuredAttrs": {
@ -28,5 +28,6 @@
], ],
"system": "my-system" "system": "my-system"
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

15
src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json
View file

@ -10,14 +10,14 @@
"out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
"out" "out"
] ]
}, },
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
@ -26,18 +26,18 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
], ],
"name": "advanced-attributes-structured-attrs", "name": "advanced-attributes-structured-attrs",
"outputs": { "outputs": {
"bin": { "bin": {
"path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" "path": "33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin"
}, },
"dev": { "dev": {
"path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" "path": "wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev"
}, },
"out": { "out": {
"path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" "path": "7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
} }
}, },
"structuredAttrs": { "structuredAttrs": {
@ -95,5 +95,6 @@
], ],
"system": "my-system" "system": "my-system"
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

11
src/libstore-tests/data/derivation/ia/advanced-attributes.json
View file

@ -24,14 +24,14 @@
"system": "my-system" "system": "my-system"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
"out" "out"
] ]
}, },
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"dev", "dev",
@ -40,13 +40,14 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
], ],
"name": "advanced-attributes", "name": "advanced-attributes",
"outputs": { "outputs": {
"out": { "out": {
"path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" "path": "wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes"
} }
}, },
"system": "my-system" "system": "my-system",
"version": 3
} }

3
src/libstore-tests/data/derivation/output-caFixedFlat.json
View file

@ -1,6 +1,5 @@
{ {
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256", "hashAlgo": "sha256",
"method": "flat", "method": "flat"
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
} }

3
src/libstore-tests/data/derivation/output-caFixedNAR.json
View file

@ -1,6 +1,5 @@
{ {
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256", "hashAlgo": "sha256",
"method": "nar", "method": "nar"
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
} }

3
src/libstore-tests/data/derivation/output-caFixedText.json
View file

@ -1,6 +1,5 @@
{ {
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"hashAlgo": "sha256", "hashAlgo": "sha256",
"method": "text", "method": "text"
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
} }

2
src/libstore-tests/data/derivation/output-inputAddressed.json
View file

@ -1,3 +1,3 @@
{ {
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
} }

7
src/libstore-tests/data/derivation/simple.json
View file

@ -8,7 +8,7 @@
"BIG_BAD": "WOLF" "BIG_BAD": "WOLF"
}, },
"inputDrvs": { "inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {}, "dynamicOutputs": {},
"outputs": [ "outputs": [
"cat", "cat",
@ -17,9 +17,10 @@
} }
}, },
"inputSrcs": [ "inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
], ],
"name": "simple-derivation", "name": "simple-derivation",
"outputs": {}, "outputs": {},
"system": "wasm-sel4" "system": "wasm-sel4",
"version": 3
} }

1
src/libstore-tests/data/store-path/simple.json Normal file
View file

@ -0,0 +1 @@
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"

9
src/libstore-tests/derivation-advanced-attrs.cc
View file

@ -59,7 +59,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
/* Use DRV file instead of C++ literal as source of truth. */ \ /* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \
EXPECT_EQ(got, expected); \ EXPECT_EQ(got, expected); \
}); \ }); \
} \ } \
@ -71,8 +71,7 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
[&]() -> json { \ [&]() -> json { \
/* Use DRV file instead of C++ literal as source of truth. */ \ /* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \
.toJSON(*this->store); \
}, \ }, \
[](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
@ -83,9 +82,9 @@ TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
this->readTest(NAME ".drv", [&](auto encoded) { \ this->readTest(NAME ".drv", [&](auto encoded) { \
/* Use JSON file instead of C++ literal as source of truth. */ \ /* Use JSON file instead of C++ literal as source of truth. */ \
auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \
auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ auto expected = Derivation::fromJSON(json, this->mockXpSettings); \
auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \
EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ EXPECT_EQ(got.toJSON(), expected.toJSON()); \
EXPECT_EQ(got, expected); \ EXPECT_EQ(got, expected); \
}); \ }); \
} \ } \

10
src/libstore-tests/derivation.cc
View file

@ -71,7 +71,7 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps)
{ \ { \
readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ readTest("output-" #NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \ auto encoded = json::parse(encoded_); \
DerivationOutput got = DerivationOutput::fromJSON(*store, DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ DerivationOutput got = DerivationOutput::fromJSON(DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \
DerivationOutput expected{VAL}; \ DerivationOutput expected{VAL}; \
ASSERT_EQ(got, expected); \ ASSERT_EQ(got, expected); \
}); \ }); \
@ -81,7 +81,7 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps)
{ \ { \
writeTest( \ writeTest( \
"output-" #NAME ".json", \ "output-" #NAME ".json", \
[&]() -> json { return DerivationOutput{(VAL)}.toJSON(*store, (DRV_NAME), (OUTPUT_NAME)); }, \ [&]() -> json { return DerivationOutput{(VAL)}.toJSON((DRV_NAME), (OUTPUT_NAME)); }, \
[](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
} }
@ -164,7 +164,7 @@ TEST_JSON(
readTest(#NAME ".json", [&](const auto & encoded_) { \ readTest(#NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \ auto encoded = json::parse(encoded_); \
Derivation expected{VAL}; \ Derivation expected{VAL}; \
Derivation got = Derivation::fromJSON(*store, encoded, mockXpSettings); \ Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \
ASSERT_EQ(got, expected); \ ASSERT_EQ(got, expected); \
}); \ }); \
} \ } \
@ -173,7 +173,7 @@ TEST_JSON(
{ \ { \
writeTest( \ writeTest( \
#NAME ".json", \ #NAME ".json", \
[&]() -> json { return Derivation{VAL}.toJSON(*store); }, \ [&]() -> json { return Derivation{VAL}.toJSON(); }, \
[](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
} }
@ -184,7 +184,7 @@ TEST_JSON(
readTest(#NAME ".drv", [&](auto encoded) { \ readTest(#NAME ".drv", [&](auto encoded) { \
Derivation expected{VAL}; \ Derivation expected{VAL}; \
auto got = parseDerivation(*store, std::move(encoded), DRV_NAME, mockXpSettings); \ auto got = parseDerivation(*store, std::move(encoded), DRV_NAME, mockXpSettings); \
ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ ASSERT_EQ(got.toJSON(), expected.toJSON()); \
ASSERT_EQ(got, expected); \ ASSERT_EQ(got, expected); \
}); \ }); \
} \ } \

44
src/libstore-tests/path.cc
View file

@ -7,7 +7,7 @@
#include "nix/store/path-regex.hh" #include "nix/store/path-regex.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/tests/hash.hh" #include "nix/util/tests/characterization.hh"
#include "nix/store/tests/libstore.hh" #include "nix/store/tests/libstore.hh"
#include "nix/store/tests/path.hh" #include "nix/store/tests/path.hh"
@ -16,8 +16,17 @@ namespace nix {
#define STORE_DIR "/nix/store/" #define STORE_DIR "/nix/store/"
#define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" #define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q"
class StorePathTest : public LibStoreTest class StorePathTest : public CharacterizationTest, public LibStoreTest
{}; {
std::filesystem::path unitTestData = getUnitTestData() / "store-path";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
static std::regex nameRegex{std::string{nameRegexStr}}; static std::regex nameRegex{std::string{nameRegexStr}};
@ -134,4 +143,33 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ())
#endif #endif
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
using nlohmann::json;
#define TEST_JSON(FIXTURE, NAME, VAL) \
static const StorePath NAME = VAL; \
\
TEST_F(FIXTURE, NAME##_from_json) \
{ \
readTest(#NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
StorePath got = static_cast<StorePath>(encoded); \
ASSERT_EQ(got, NAME); \
}); \
} \
\
TEST_F(FIXTURE, NAME##_to_json) \
{ \
writeTest( \
#NAME ".json", \
[&]() -> json { return static_cast<json>(NAME); }, \
[](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
}
TEST_JSON(StorePathTest, simple, StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"});
} // namespace nix } // namespace nix

54
src/libstore/derivations.cc
View file

@ -1257,15 +1257,14 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
nlohmann::json nlohmann::json DerivationOutput::toJSON(std::string_view drvName, OutputNameView outputName) const
DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
{ {
nlohmann::json res = nlohmann::json::object(); nlohmann::json res = nlohmann::json::object();
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOutput::InputAddressed & doi) { res["path"] = store.printStorePath(doi.path); }, [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); // res["path"] = dof.path(store, drvName, outputName);
res["method"] = std::string{dof.ca.method.render()}; res["method"] = std::string{dof.ca.method.render()};
res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo);
res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
@ -1287,7 +1286,6 @@ DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName,
} }
DerivationOutput DerivationOutput::fromJSON( DerivationOutput DerivationOutput::fromJSON(
const StoreDirConfig & store,
std::string_view drvName, std::string_view drvName,
OutputNameView outputName, OutputNameView outputName,
const nlohmann::json & _json, const nlohmann::json & _json,
@ -1310,11 +1308,11 @@ DerivationOutput DerivationOutput::fromJSON(
if (keys == (std::set<std::string_view>{"path"})) { if (keys == (std::set<std::string_view>{"path"})) {
return DerivationOutput::InputAddressed{ return DerivationOutput::InputAddressed{
.path = store.parseStorePath(getString(valueAt(json, "path"))), .path = valueAt(json, "path"),
}; };
} }
else if (keys == (std::set<std::string_view>{"path", "method", "hashAlgo", "hash"})) { else if (keys == (std::set<std::string_view>{"method", "hashAlgo", "hash"})) {
auto [method, hashAlgo] = methodAlgo(); auto [method, hashAlgo] = methodAlgo();
auto dof = DerivationOutput::CAFixed{ auto dof = DerivationOutput::CAFixed{
.ca = .ca =
@ -1323,8 +1321,10 @@ DerivationOutput DerivationOutput::fromJSON(
.hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo),
}, },
}; };
if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) #if 0
if (dof.path(store, drvName, outputName) != static_cast<StorePath>(valueAt(json, "path")))
throw Error("Path doesn't match derivation output"); throw Error("Path doesn't match derivation output");
#endif
return dof; return dof;
} }
@ -1355,17 +1355,19 @@ DerivationOutput DerivationOutput::fromJSON(
} }
} }
nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const nlohmann::json Derivation::toJSON() const
{ {
nlohmann::json res = nlohmann::json::object(); nlohmann::json res = nlohmann::json::object();
res["name"] = name; res["name"] = name;
res["version"] = 3;
{ {
nlohmann::json & outputsObj = res["outputs"]; nlohmann::json & outputsObj = res["outputs"];
outputsObj = nlohmann::json::object(); outputsObj = nlohmann::json::object();
for (auto & [outputName, output] : outputs) { for (auto & [outputName, output] : outputs) {
outputsObj[outputName] = output.toJSON(store, name, outputName); outputsObj[outputName] = output.toJSON(name, outputName);
} }
} }
@ -1373,7 +1375,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
auto & inputsList = res["inputSrcs"]; auto & inputsList = res["inputSrcs"];
inputsList = nlohmann::json ::array(); inputsList = nlohmann::json ::array();
for (auto & input : inputSrcs) for (auto & input : inputSrcs)
inputsList.emplace_back(store.printStorePath(input)); inputsList.emplace_back(input);
} }
{ {
@ -1393,7 +1395,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
auto & inputDrvsObj = res["inputDrvs"]; auto & inputDrvsObj = res["inputDrvs"];
inputDrvsObj = nlohmann::json::object(); inputDrvsObj = nlohmann::json::object();
for (auto & [inputDrv, inputNode] : inputDrvs.map) { for (auto & [inputDrv, inputNode] : inputDrvs.map) {
inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
} }
} }
} }
@ -1409,8 +1411,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
return res; return res;
} }
Derivation Derivation::fromJSON( Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
const StoreDirConfig & store, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
{ {
using nlohmann::detail::value_t; using nlohmann::detail::value_t;
@ -1420,11 +1421,14 @@ Derivation Derivation::fromJSON(
res.name = getString(valueAt(json, "name")); res.name = getString(valueAt(json, "name"));
if (valueAt(json, "version") != 3)
throw Error("Only derivation format version 3 is currently supported.");
try { try {
auto outputs = getObject(valueAt(json, "outputs")); auto outputs = getObject(valueAt(json, "outputs"));
for (auto & [outputName, output] : outputs) { for (auto & [outputName, output] : outputs) {
res.outputs.insert_or_assign( res.outputs.insert_or_assign(
outputName, DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); outputName, DerivationOutput::fromJSON(res.name, outputName, output, xpSettings));
} }
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while reading key 'outputs'"); e.addTrace({}, "while reading key 'outputs'");
@ -1434,7 +1438,7 @@ Derivation Derivation::fromJSON(
try { try {
auto inputSrcs = getArray(valueAt(json, "inputSrcs")); auto inputSrcs = getArray(valueAt(json, "inputSrcs"));
for (auto & input : inputSrcs) for (auto & input : inputSrcs)
res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input))); res.inputSrcs.insert(input);
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while reading key 'inputSrcs'"); e.addTrace({}, "while reading key 'inputSrcs'");
throw; throw;
@ -1455,7 +1459,7 @@ Derivation Derivation::fromJSON(
}; };
auto drvs = getObject(valueAt(json, "inputDrvs")); auto drvs = getObject(valueAt(json, "inputDrvs"));
for (auto & [inputDrvPath, inputOutputs] : drvs) for (auto & [inputDrvPath, inputOutputs] : drvs)
res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs);
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while reading key 'inputDrvs'"); e.addTrace({}, "while reading key 'inputDrvs'");
throw; throw;
@ -1480,3 +1484,19 @@ Derivation Derivation::fromJSON(
} }
} // namespace nix } // namespace nix
namespace nlohmann {
using namespace nix;
Derivation adl_serializer<Derivation>::from_json(const json & json)
{
return Derivation::fromJSON(json);
}
void adl_serializer<Derivation>::to_json(json & json, Derivation c)
{
json = c.toJSON();
}
} // namespace nlohmann

13
src/libstore/include/nix/store/derivations.hh
View file

@ -135,12 +135,11 @@ struct DerivationOutput
std::optional<StorePath> std::optional<StorePath>
path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
nlohmann::json toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON(std::string_view drvName, OutputNameView outputName) const;
/** /**
* @param xpSettings Stop-gap to avoid globals during unit tests. * @param xpSettings Stop-gap to avoid globals during unit tests.
*/ */
static DerivationOutput fromJSON( static DerivationOutput fromJSON(
const StoreDirConfig & store,
std::string_view drvName, std::string_view drvName,
OutputNameView outputName, OutputNameView outputName,
const nlohmann::json & json, const nlohmann::json & json,
@ -394,11 +393,9 @@ struct Derivation : BasicDerivation
{ {
} }
nlohmann::json toJSON(const StoreDirConfig & store) const; nlohmann::json toJSON() const;
static Derivation fromJSON( static Derivation
const StoreDirConfig & store, fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
const nlohmann::json & json,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
bool operator==(const Derivation &) const = default; bool operator==(const Derivation &) const = default;
// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
@ -542,3 +539,5 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
std::string hashPlaceholder(const OutputNameView outputName); std::string hashPlaceholder(const OutputNameView outputName);
} // namespace nix } // namespace nix
JSON_IMPL(nix::Derivation)

8
src/libstore/include/nix/store/path.hh
View file

@ -4,6 +4,8 @@
#include <string_view> #include <string_view>
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/json-impls.hh"
#include "nix/util/json-non-null.hh"
namespace nix { namespace nix {
@ -87,6 +89,10 @@ typedef std::vector<StorePath> StorePaths;
*/ */
constexpr std::string_view drvExtension = ".drv"; constexpr std::string_view drvExtension = ".drv";
template<>
struct json_avoids_null<StorePath> : std::true_type
{};
} // namespace nix } // namespace nix
namespace std { namespace std {
@ -101,3 +107,5 @@ struct hash<nix::StorePath>
}; };
} // namespace std } // namespace std
JSON_IMPL(nix::StorePath)

19
src/libstore/path.cc
View file

@ -1,4 +1,7 @@
#include <nlohmann/json.hpp>
#include "nix/store/store-dir-config.hh" #include "nix/store/store-dir-config.hh"
#include "nix/util/json-utils.hh"
namespace nix { namespace nix {
@ -75,3 +78,19 @@ StorePath StorePath::random(std::string_view name)
} }
} // namespace nix } // namespace nix
namespace nlohmann {
using namespace nix;
StorePath adl_serializer<StorePath>::from_json(const json & json)
{
return StorePath{getString(json)};
}
void adl_serializer<StorePath>::to_json(json & json, StorePath storePath)
{
json = storePath.to_string();
}
} // namespace nlohmann

2
src/nix/derivation-add.cc
View file

@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand
{ {
auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); auto json = nlohmann::json::parse(drainFD(STDIN_FILENO));
auto drv = Derivation::fromJSON(*store, json); auto drv = Derivation::fromJSON(json);
auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun);

2
src/nix/derivation-show.cc
View file

@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON
if (!drvPath.isDerivation()) if (!drvPath.isDerivation())
continue; continue;
jsonRoot[store->printStorePath(drvPath)] = store->readDerivation(drvPath).toJSON(*store); jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON();
} }
printJSON(jsonRoot); printJSON(jsonRoot);
} }

9
tests/functional/dyn-drv/non-trivial.nix
View file

@ -62,12 +62,15 @@ builtins.outputOf
"hashAlgo": "sha256" "hashAlgo": "sha256"
} }
}, },
"system": "${system}" "system": "${system}",
"version": 3
} }
EOF EOF
drvs[$word]="$(echo "$json" | nix derivation add)" drvPath=$(echo "$json" | nix derivation add)
storeDir=$(dirname "$drvPath")
drvs[$word]="$(basename "$drvPath")"
done done
cp "''${drvs[e]}" $out cp "''${storeDir}/''${drvs[e]}" $out
''; '';
__contentAddressed = true; __contentAddressed = true;

4
tests/functional/impure-derivations.sh
View file

@ -50,8 +50,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] [[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]]
[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] [[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]]
# Fixed-output derivations *can* depend on impure derivations. # Fixed-output derivations *can* depend on impure derivations.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)

2
tests/functional/structured-attrs.sh
View file

@ -50,4 +50,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "
# Check it works with the expected structured attrs # Check it works with the expected structured attrs
hacky=$(nix-instantiate --expr "$hackyExpr") hacky=$(nix-instantiate --expr "$hackyExpr")
nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}' nix derivation show "$hacky" | jq --exit-status '."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'