From 9d7229a2a429b7de0e392d40f222d3d2802989da Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Sep 2025 08:25:42 -0400 Subject: [PATCH] Make the JSON format for derivation use basename store paths See #13570 for details --- the idea is that included the store dir in store paths makes systematic JSON parting with e.g. Serde, Aeson, nlohmann, or similiar harder. After talking to Eelco, we are changing the `Derivation` format right away because not only is `nix derivation` technically experimental, we think it is also less widely used in practice than, say, `nix path-info`. Progress on #13570 --- doc/manual/rl-next/derivation-json.md | 17 ++++ .../source/protocols/json/derivation.md | 29 ++++++- src/libstore-c/nix_api_store.cc | 2 +- .../ca/advanced-attributes-defaults.json | 3 +- ...-attributes-structured-attrs-defaults.json | 3 +- .../advanced-attributes-structured-attrs.json | 9 ++- .../derivation/ca/advanced-attributes.json | 9 ++- .../data/derivation/ca/self-contained.json | 3 +- .../data/derivation/dynDerivationDeps.json | 7 +- .../ia/advanced-attributes-defaults.json | 5 +- ...-attributes-structured-attrs-defaults.json | 7 +- .../advanced-attributes-structured-attrs.json | 15 ++-- .../derivation/ia/advanced-attributes.json | 11 +-- .../data/derivation/output-caFixedFlat.json | 3 +- .../data/derivation/output-caFixedNAR.json | 3 +- .../data/derivation/output-caFixedText.json | 3 +- .../derivation/output-inputAddressed.json | 2 +- .../data/derivation/simple.json | 7 +- .../data/store-path/simple.json | 1 + .../derivation-advanced-attrs.cc | 77 +++++++++---------- src/libstore-tests/derivation.cc | 42 +++++----- src/libstore-tests/path.cc | 44 ++++++++++- src/libstore/derivations.cc | 54 +++++++++---- src/libstore/include/nix/store/derivations.hh | 13 ++-- src/libstore/include/nix/store/path.hh | 8 ++ src/libstore/path.cc | 19 +++++ src/nix/derivation-add.cc | 2 +- src/nix/derivation-show.cc | 2 +- tests/functional/dyn-drv/non-trivial.nix | 9 ++- tests/functional/impure-derivations.sh | 4 +- tests/functional/structured-attrs.sh | 2 +- 31 files changed, 275 insertions(+), 140 deletions(-) create mode 100644 doc/manual/rl-next/derivation-json.md create mode 100644 src/libstore-tests/data/store-path/simple.json diff --git a/doc/manual/rl-next/derivation-json.md b/doc/manual/rl-next/derivation-json.md new file mode 100644 index 000000000..420395f1d --- /dev/null +++ b/doc/manual/rl-next/derivation-json.md @@ -0,0 +1,17 @@ +--- +synopsis: Derivation JSON format now uses store path basenames (no store dir) only +prs: [13980] +issues: [13570] +--- + +Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde +in Rust, and Aeson in Haskell), has show that the use of the store dir +in JSON formats is an impediment to systematic JSON formats, because it +requires the serializer/deserializer to take an extra paramater (the +store dir). + +We ultimately want to rectify this issue with all (non-stable, able to +be changed) JSON formats. To start with, we are changing the JSON format +for derivations because the `nix derivation` commands are --- in +addition to being formally unstable --- less widely used than other +unstable commands. diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 04881776a..566288962 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -14,6 +14,21 @@ is a JSON object with the following fields: The name of the derivation. This is used when calculating the store paths of the derivation's outputs. +* `version`: + Must be `3`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: + + - Version 0: A-Term format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Verison 3: Drop store dir from store paths, just include base name. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: @@ -52,7 +67,6 @@ is a JSON object with the following fields: > ```json > "outputs": { > "out": { - > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", > "method": "nar", > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" @@ -63,6 +77,15 @@ is a JSON object with the following fields: * `inputSrcs`: A list of store paths on which this derivation depends. + > **Example** + > + > ```json + > "inputSrcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + * `inputDrvs`: A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. @@ -70,8 +93,8 @@ is a JSON object with the following fields: > > ```json > "inputDrvs": { - > "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] > } > ``` diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index a319c0c10..c4c17f127 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -181,7 +181,7 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store if (context) context->last_err_code = NIX_OK; try { - auto drv = nix::Derivation::fromJSON(*store->ptr, nlohmann::json::parse(json)); + auto drv = static_cast(nlohmann::json::parse(json)); auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true); diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index bc67236b5..eb4bd4f3d 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -21,5 +21,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 183148b29..3a4a3079b 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -32,5 +32,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index ec044d778..b10355af7 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,7 +26,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { @@ -100,5 +100,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 0ac0a9c5c..d66882036 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -26,14 +26,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -42,7 +42,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes", "outputs": { @@ -51,5 +51,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index c4ca280ef..331beb7be 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -19,5 +19,6 @@ "method": "nar" } }, - "system": "x86_64-linux" + "system": "x86_64-linux", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/dynDerivationDeps.json b/src/libstore-tests/data/derivation/dynDerivationDeps.json index 9dbeb1f15..1a9f54c53 100644 --- a/src/libstore-tests/data/derivation/dynDerivationDeps.json +++ b/src/libstore-tests/data/derivation/dynDerivationDeps.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": { "cat": { "dynamicOutputs": {}, @@ -30,9 +30,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "dyn-dep-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index d58e7d5b5..0fa543f21 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -15,8 +15,9 @@ "name": "advanced-attributes-defaults", "outputs": { "out": { - "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + "path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index f5349e6c3..e02392ea1 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -13,10 +13,10 @@ "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { - "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + "path": "8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" }, "out": { - "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + "path": "f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" } }, "structuredAttrs": { @@ -28,5 +28,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index b8d566462..9230b06b6 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,18 +26,18 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" + "path": "33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" + "path": "wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" + "path": "7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, "structuredAttrs": { @@ -95,5 +95,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index 20ce5e1c2..ba5911c91 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -24,14 +24,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -40,13 +40,14 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" + "path": "wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index 7001ea0a9..e6a0123f6 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "flat", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" + "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index 54eb306e6..b57e065a9 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "nar", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index e8a651860..84778509e 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "text", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + "method": "text" } diff --git a/src/libstore-tests/data/derivation/output-inputAddressed.json b/src/libstore-tests/data/derivation/output-inputAddressed.json index 86c7f3a05..04491ffde 100644 --- a/src/libstore-tests/data/derivation/output-inputAddressed.json +++ b/src/libstore-tests/data/derivation/output-inputAddressed.json @@ -1,3 +1,3 @@ { - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" } diff --git a/src/libstore-tests/data/derivation/simple.json b/src/libstore-tests/data/derivation/simple.json index 20d0f8933..41a049aef 100644 --- a/src/libstore-tests/data/derivation/simple.json +++ b/src/libstore-tests/data/derivation/simple.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": {}, "outputs": [ "cat", @@ -17,9 +17,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "simple-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/store-path/simple.json b/src/libstore-tests/data/store-path/simple.json new file mode 100644 index 000000000..9bedb882b --- /dev/null +++ b/src/libstore-tests/data/store-path/simple.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 37b422421..9c13bf048 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -51,45 +51,44 @@ using BothFixtures = ::testing::TypesreadTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ - Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ - { \ - this->writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ - .toJSON(*this->store); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ - { \ - this->readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ - auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ - EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(json, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(got.toJSON(), expected.toJSON()); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); diff --git a/src/libstore-tests/derivation.cc b/src/libstore-tests/derivation.cc index 812e1d01b..35992c5ec 100644 --- a/src/libstore-tests/derivation.cc +++ b/src/libstore-tests/derivation.cc @@ -66,24 +66,24 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) FormatError); } -#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_from_json) \ - { \ - readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - DerivationOutput got = DerivationOutput::fromJSON(*store, DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ - DerivationOutput expected{VAL}; \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_to_json) \ - { \ - writeTest( \ - "output-" #NAME ".json", \ - [&]() -> json { return DerivationOutput{(VAL)}.toJSON(*store, (DRV_NAME), (OUTPUT_NAME)); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_##NAME##_from_json) \ + { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON(DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ + DerivationOutput expected{VAL}; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_##NAME##_to_json) \ + { \ + writeTest( \ + "output-" #NAME ".json", \ + [&]() -> json { return DerivationOutput{(VAL)}.toJSON((DRV_NAME), (OUTPUT_NAME)); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } TEST_JSON( @@ -164,7 +164,7 @@ TEST_JSON( readTest(#NAME ".json", [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ Derivation expected{VAL}; \ - Derivation got = Derivation::fromJSON(*store, encoded, mockXpSettings); \ + Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \ ASSERT_EQ(got, expected); \ }); \ } \ @@ -173,7 +173,7 @@ TEST_JSON( { \ writeTest( \ #NAME ".json", \ - [&]() -> json { return Derivation{VAL}.toJSON(*store); }, \ + [&]() -> json { return Derivation{VAL}.toJSON(); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } @@ -184,7 +184,7 @@ TEST_JSON( readTest(#NAME ".drv", [&](auto encoded) { \ Derivation expected{VAL}; \ auto got = parseDerivation(*store, std::move(encoded), DRV_NAME, mockXpSettings); \ - ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + ASSERT_EQ(got.toJSON(), expected.toJSON()); \ ASSERT_EQ(got, expected); \ }); \ } \ diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index 01d1ca792..b6a1a541f 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -7,7 +7,7 @@ #include "nix/store/path-regex.hh" #include "nix/store/store-api.hh" -#include "nix/util/tests/hash.hh" +#include "nix/util/tests/characterization.hh" #include "nix/store/tests/libstore.hh" #include "nix/store/tests/path.hh" @@ -16,8 +16,17 @@ namespace nix { #define STORE_DIR "/nix/store/" #define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" -class StorePathTest : public LibStoreTest -{}; +class StorePathTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "store-path"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; static std::regex nameRegex{std::string{nameRegexStr}}; @@ -134,4 +143,33 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) #endif +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + static const StorePath NAME = VAL; \ + \ + TEST_F(FIXTURE, NAME##_from_json) \ + { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StorePath got = static_cast(encoded); \ + ASSERT_EQ(got, NAME); \ + }); \ + } \ + \ + TEST_F(FIXTURE, NAME##_to_json) \ + { \ + writeTest( \ + #NAME ".json", \ + [&]() -> json { return static_cast(NAME); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +TEST_JSON(StorePathTest, simple, StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}); + } // namespace nix diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 84889ceac..92266b61b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1257,15 +1257,14 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); -nlohmann::json -DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const +nlohmann::json DerivationOutput::toJSON(std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit( overloaded{ - [&](const DerivationOutput::InputAddressed & doi) { res["path"] = store.printStorePath(doi.path); }, + [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + // res["path"] = dof.path(store, drvName, outputName); res["method"] = std::string{dof.ca.method.render()}; res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); @@ -1287,7 +1286,6 @@ DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, } DerivationOutput DerivationOutput::fromJSON( - const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, @@ -1310,11 +1308,11 @@ DerivationOutput DerivationOutput::fromJSON( if (keys == (std::set{"path"})) { return DerivationOutput::InputAddressed{ - .path = store.parseStorePath(getString(valueAt(json, "path"))), + .path = valueAt(json, "path"), }; } - else if (keys == (std::set{"path", "method", "hashAlgo", "hash"})) { + else if (keys == (std::set{"method", "hashAlgo", "hash"})) { auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed{ .ca = @@ -1323,8 +1321,10 @@ DerivationOutput DerivationOutput::fromJSON( .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), }, }; - if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) +#if 0 + if (dof.path(store, drvName, outputName) != static_cast(valueAt(json, "path"))) throw Error("Path doesn't match derivation output"); +#endif return dof; } @@ -1355,17 +1355,19 @@ DerivationOutput DerivationOutput::fromJSON( } } -nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const +nlohmann::json Derivation::toJSON() const { nlohmann::json res = nlohmann::json::object(); res["name"] = name; + res["version"] = 3; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); for (auto & [outputName, output] : outputs) { - outputsObj[outputName] = output.toJSON(store, name, outputName); + outputsObj[outputName] = output.toJSON(name, outputName); } } @@ -1373,7 +1375,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputsList = res["inputSrcs"]; inputsList = nlohmann::json ::array(); for (auto & input : inputSrcs) - inputsList.emplace_back(store.printStorePath(input)); + inputsList.emplace_back(input); } { @@ -1393,7 +1395,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputDrvsObj = res["inputDrvs"]; inputDrvsObj = nlohmann::json::object(); for (auto & [inputDrv, inputNode] : inputDrvs.map) { - inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } } @@ -1409,8 +1411,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const return res; } -Derivation Derivation::fromJSON( - const StoreDirConfig & store, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) +Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1420,11 +1421,14 @@ Derivation Derivation::fromJSON( res.name = getString(valueAt(json, "name")); + if (valueAt(json, "version") != 3) + throw Error("Only derivation format version 3 is currently supported."); + try { auto outputs = getObject(valueAt(json, "outputs")); for (auto & [outputName, output] : outputs) { res.outputs.insert_or_assign( - outputName, DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); + outputName, DerivationOutput::fromJSON(res.name, outputName, output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); @@ -1434,7 +1438,7 @@ Derivation Derivation::fromJSON( try { auto inputSrcs = getArray(valueAt(json, "inputSrcs")); for (auto & input : inputSrcs) - res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + res.inputSrcs.insert(input); } catch (Error & e) { e.addTrace({}, "while reading key 'inputSrcs'"); throw; @@ -1455,7 +1459,7 @@ Derivation Derivation::fromJSON( }; auto drvs = getObject(valueAt(json, "inputDrvs")); for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); } catch (Error & e) { e.addTrace({}, "while reading key 'inputDrvs'"); throw; @@ -1480,3 +1484,19 @@ Derivation Derivation::fromJSON( } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +Derivation adl_serializer::from_json(const json & json) +{ + return Derivation::fromJSON(json); +} + +void adl_serializer::to_json(json & json, Derivation c) +{ + json = c.toJSON(); +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 08bb7183f..d66bcef2e 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -135,12 +135,11 @@ struct DerivationOutput std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - nlohmann::json toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; + nlohmann::json toJSON(std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -394,11 +393,9 @@ struct Derivation : BasicDerivation { } - nlohmann::json toJSON(const StoreDirConfig & store) const; - static Derivation fromJSON( - const StoreDirConfig & store, - const nlohmann::json & json, - const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON() const; + static Derivation + fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); bool operator==(const Derivation &) const = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. @@ -542,3 +539,5 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName); } // namespace nix + +JSON_IMPL(nix::Derivation) diff --git a/src/libstore/include/nix/store/path.hh b/src/libstore/include/nix/store/path.hh index 784298daa..8124cf580 100644 --- a/src/libstore/include/nix/store/path.hh +++ b/src/libstore/include/nix/store/path.hh @@ -4,6 +4,8 @@ #include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/json-non-null.hh" namespace nix { @@ -87,6 +89,10 @@ typedef std::vector StorePaths; */ constexpr std::string_view drvExtension = ".drv"; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix namespace std { @@ -101,3 +107,5 @@ struct hash }; } // namespace std + +JSON_IMPL(nix::StorePath) diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 516b01571..942f97a88 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,7 @@ +#include + #include "nix/store/store-dir-config.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -75,3 +78,19 @@ StorePath StorePath::random(std::string_view name) } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +StorePath adl_serializer::from_json(const json & json) +{ + return StorePath{getString(json)}; +} + +void adl_serializer::to_json(json & json, StorePath storePath) +{ + json = storePath.to_string(); +} + +} // namespace nlohmann diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 0f797bb20..2d13aba52 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand { auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); - auto drv = Derivation::fromJSON(*store, json); + auto drv = Derivation::fromJSON(json); auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 1a61ccd5c..20e54bba7 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON if (!drvPath.isDerivation()) continue; - jsonRoot[store->printStorePath(drvPath)] = store->readDerivation(drvPath).toJSON(*store); + jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON(); } printJSON(jsonRoot); } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 5cfafbb62..3c24ac2ee 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -62,12 +62,15 @@ builtins.outputOf "hashAlgo": "sha256" } }, - "system": "${system}" + "system": "${system}", + "version": 3 } EOF - drvs[$word]="$(echo "$json" | nix derivation add)" + drvPath=$(echo "$json" | nix derivation add) + storeDir=$(dirname "$drvPath") + drvs[$word]="$(basename "$drvPath")" done - cp "''${drvs[e]}" $out + cp "''${storeDir}/''${drvs[e]}" $out ''; __contentAddressed = true; diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 5dea220fe..9e483d376 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -50,8 +50,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 2bd9b4aaf..dfd5a1412 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -50,4 +50,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") -nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}' +nix derivation show "$hacky" | jq --exit-status '."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'