diff --git a/doc/manual/rl-next/derivation-json.md b/doc/manual/rl-next/derivation-json.md new file mode 100644 index 000000000..420395f1d --- /dev/null +++ b/doc/manual/rl-next/derivation-json.md @@ -0,0 +1,17 @@ +--- +synopsis: Derivation JSON format now uses store path basenames (no store dir) only +prs: [13980] +issues: [13570] +--- + +Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde +in Rust, and Aeson in Haskell), has show that the use of the store dir +in JSON formats is an impediment to systematic JSON formats, because it +requires the serializer/deserializer to take an extra paramater (the +store dir). + +We ultimately want to rectify this issue with all (non-stable, able to +be changed) JSON formats. To start with, we are changing the JSON format +for derivations because the `nix derivation` commands are --- in +addition to being formally unstable --- less widely used than other +unstable commands. diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 04881776a..566288962 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -14,6 +14,21 @@ is a JSON object with the following fields: The name of the derivation. This is used when calculating the store paths of the derivation's outputs. +* `version`: + Must be `3`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: + + - Version 0: A-Term format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Verison 3: Drop store dir from store paths, just include base name. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: @@ -52,7 +67,6 @@ is a JSON object with the following fields: > ```json > "outputs": { > "out": { - > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", > "method": "nar", > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" @@ -63,6 +77,15 @@ is a JSON object with the following fields: * `inputSrcs`: A list of store paths on which this derivation depends. + > **Example** + > + > ```json + > "inputSrcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + * `inputDrvs`: A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. @@ -70,8 +93,8 @@ is a JSON object with the following fields: > > ```json > "inputDrvs": { - > "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] > } > ``` diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index a319c0c10..c4c17f127 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -181,7 +181,7 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store if (context) context->last_err_code = NIX_OK; try { - auto drv = nix::Derivation::fromJSON(*store->ptr, nlohmann::json::parse(json)); + auto drv = static_cast(nlohmann::json::parse(json)); auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true); diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index bc67236b5..eb4bd4f3d 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -21,5 +21,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 183148b29..3a4a3079b 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -32,5 +32,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index ec044d778..b10355af7 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,7 +26,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { @@ -100,5 +100,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 0ac0a9c5c..d66882036 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -26,14 +26,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -42,7 +42,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes", "outputs": { @@ -51,5 +51,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json index c4ca280ef..331beb7be 100644 --- a/src/libstore-tests/data/derivation/ca/self-contained.json +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -19,5 +19,6 @@ "method": "nar" } }, - "system": "x86_64-linux" + "system": "x86_64-linux", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/dynDerivationDeps.json b/src/libstore-tests/data/derivation/dynDerivationDeps.json index 9dbeb1f15..1a9f54c53 100644 --- a/src/libstore-tests/data/derivation/dynDerivationDeps.json +++ b/src/libstore-tests/data/derivation/dynDerivationDeps.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": { "cat": { "dynamicOutputs": {}, @@ -30,9 +30,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "dyn-dep-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index d58e7d5b5..0fa543f21 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -15,8 +15,9 @@ "name": "advanced-attributes-defaults", "outputs": { "out": { - "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + "path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index f5349e6c3..e02392ea1 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -13,10 +13,10 @@ "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { - "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + "path": "8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" }, "out": { - "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + "path": "f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" } }, "structuredAttrs": { @@ -28,5 +28,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index b8d566462..9230b06b6 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,18 +26,18 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" + "path": "33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" + "path": "wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" + "path": "7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, "structuredAttrs": { @@ -95,5 +95,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index 20ce5e1c2..ba5911c91 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -24,14 +24,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -40,13 +40,14 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" + "path": "wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index 7001ea0a9..e6a0123f6 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "flat", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" + "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index 54eb306e6..b57e065a9 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "nar", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index e8a651860..84778509e 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "text", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + "method": "text" } diff --git a/src/libstore-tests/data/derivation/output-inputAddressed.json b/src/libstore-tests/data/derivation/output-inputAddressed.json index 86c7f3a05..04491ffde 100644 --- a/src/libstore-tests/data/derivation/output-inputAddressed.json +++ b/src/libstore-tests/data/derivation/output-inputAddressed.json @@ -1,3 +1,3 @@ { - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" } diff --git a/src/libstore-tests/data/derivation/simple.json b/src/libstore-tests/data/derivation/simple.json index 20d0f8933..41a049aef 100644 --- a/src/libstore-tests/data/derivation/simple.json +++ b/src/libstore-tests/data/derivation/simple.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": {}, "outputs": [ "cat", @@ -17,9 +17,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "simple-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/store-path/simple.json b/src/libstore-tests/data/store-path/simple.json new file mode 100644 index 000000000..9bedb882b --- /dev/null +++ b/src/libstore-tests/data/store-path/simple.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 37b422421..9c13bf048 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -51,45 +51,44 @@ using BothFixtures = ::testing::TypesreadTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ - Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ - { \ - this->writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ - .toJSON(*this->store); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ - { \ - this->readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ - auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ - EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(json, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(got.toJSON(), expected.toJSON()); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); diff --git a/src/libstore-tests/derivation.cc b/src/libstore-tests/derivation.cc index 812e1d01b..35992c5ec 100644 --- a/src/libstore-tests/derivation.cc +++ b/src/libstore-tests/derivation.cc @@ -66,24 +66,24 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) FormatError); } -#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_from_json) \ - { \ - readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - DerivationOutput got = DerivationOutput::fromJSON(*store, DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ - DerivationOutput expected{VAL}; \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_to_json) \ - { \ - writeTest( \ - "output-" #NAME ".json", \ - [&]() -> json { return DerivationOutput{(VAL)}.toJSON(*store, (DRV_NAME), (OUTPUT_NAME)); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_##NAME##_from_json) \ + { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON(DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ + DerivationOutput expected{VAL}; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_##NAME##_to_json) \ + { \ + writeTest( \ + "output-" #NAME ".json", \ + [&]() -> json { return DerivationOutput{(VAL)}.toJSON((DRV_NAME), (OUTPUT_NAME)); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } TEST_JSON( @@ -164,7 +164,7 @@ TEST_JSON( readTest(#NAME ".json", [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ Derivation expected{VAL}; \ - Derivation got = Derivation::fromJSON(*store, encoded, mockXpSettings); \ + Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \ ASSERT_EQ(got, expected); \ }); \ } \ @@ -173,7 +173,7 @@ TEST_JSON( { \ writeTest( \ #NAME ".json", \ - [&]() -> json { return Derivation{VAL}.toJSON(*store); }, \ + [&]() -> json { return Derivation{VAL}.toJSON(); }, \ [](const auto & file) { return json::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ } @@ -184,7 +184,7 @@ TEST_JSON( readTest(#NAME ".drv", [&](auto encoded) { \ Derivation expected{VAL}; \ auto got = parseDerivation(*store, std::move(encoded), DRV_NAME, mockXpSettings); \ - ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + ASSERT_EQ(got.toJSON(), expected.toJSON()); \ ASSERT_EQ(got, expected); \ }); \ } \ diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index 01d1ca792..b6a1a541f 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -7,7 +7,7 @@ #include "nix/store/path-regex.hh" #include "nix/store/store-api.hh" -#include "nix/util/tests/hash.hh" +#include "nix/util/tests/characterization.hh" #include "nix/store/tests/libstore.hh" #include "nix/store/tests/path.hh" @@ -16,8 +16,17 @@ namespace nix { #define STORE_DIR "/nix/store/" #define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" -class StorePathTest : public LibStoreTest -{}; +class StorePathTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "store-path"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; static std::regex nameRegex{std::string{nameRegexStr}}; @@ -134,4 +143,33 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) #endif +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + static const StorePath NAME = VAL; \ + \ + TEST_F(FIXTURE, NAME##_from_json) \ + { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StorePath got = static_cast(encoded); \ + ASSERT_EQ(got, NAME); \ + }); \ + } \ + \ + TEST_F(FIXTURE, NAME##_to_json) \ + { \ + writeTest( \ + #NAME ".json", \ + [&]() -> json { return static_cast(NAME); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +TEST_JSON(StorePathTest, simple, StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}); + } // namespace nix diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 84889ceac..92266b61b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1257,15 +1257,14 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); -nlohmann::json -DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const +nlohmann::json DerivationOutput::toJSON(std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit( overloaded{ - [&](const DerivationOutput::InputAddressed & doi) { res["path"] = store.printStorePath(doi.path); }, + [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + // res["path"] = dof.path(store, drvName, outputName); res["method"] = std::string{dof.ca.method.render()}; res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); @@ -1287,7 +1286,6 @@ DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, } DerivationOutput DerivationOutput::fromJSON( - const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, @@ -1310,11 +1308,11 @@ DerivationOutput DerivationOutput::fromJSON( if (keys == (std::set{"path"})) { return DerivationOutput::InputAddressed{ - .path = store.parseStorePath(getString(valueAt(json, "path"))), + .path = valueAt(json, "path"), }; } - else if (keys == (std::set{"path", "method", "hashAlgo", "hash"})) { + else if (keys == (std::set{"method", "hashAlgo", "hash"})) { auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed{ .ca = @@ -1323,8 +1321,10 @@ DerivationOutput DerivationOutput::fromJSON( .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), }, }; - if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) +#if 0 + if (dof.path(store, drvName, outputName) != static_cast(valueAt(json, "path"))) throw Error("Path doesn't match derivation output"); +#endif return dof; } @@ -1355,17 +1355,19 @@ DerivationOutput DerivationOutput::fromJSON( } } -nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const +nlohmann::json Derivation::toJSON() const { nlohmann::json res = nlohmann::json::object(); res["name"] = name; + res["version"] = 3; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); for (auto & [outputName, output] : outputs) { - outputsObj[outputName] = output.toJSON(store, name, outputName); + outputsObj[outputName] = output.toJSON(name, outputName); } } @@ -1373,7 +1375,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputsList = res["inputSrcs"]; inputsList = nlohmann::json ::array(); for (auto & input : inputSrcs) - inputsList.emplace_back(store.printStorePath(input)); + inputsList.emplace_back(input); } { @@ -1393,7 +1395,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputDrvsObj = res["inputDrvs"]; inputDrvsObj = nlohmann::json::object(); for (auto & [inputDrv, inputNode] : inputDrvs.map) { - inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } } @@ -1409,8 +1411,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const return res; } -Derivation Derivation::fromJSON( - const StoreDirConfig & store, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) +Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1420,11 +1421,14 @@ Derivation Derivation::fromJSON( res.name = getString(valueAt(json, "name")); + if (valueAt(json, "version") != 3) + throw Error("Only derivation format version 3 is currently supported."); + try { auto outputs = getObject(valueAt(json, "outputs")); for (auto & [outputName, output] : outputs) { res.outputs.insert_or_assign( - outputName, DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); + outputName, DerivationOutput::fromJSON(res.name, outputName, output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); @@ -1434,7 +1438,7 @@ Derivation Derivation::fromJSON( try { auto inputSrcs = getArray(valueAt(json, "inputSrcs")); for (auto & input : inputSrcs) - res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + res.inputSrcs.insert(input); } catch (Error & e) { e.addTrace({}, "while reading key 'inputSrcs'"); throw; @@ -1455,7 +1459,7 @@ Derivation Derivation::fromJSON( }; auto drvs = getObject(valueAt(json, "inputDrvs")); for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); } catch (Error & e) { e.addTrace({}, "while reading key 'inputDrvs'"); throw; @@ -1480,3 +1484,19 @@ Derivation Derivation::fromJSON( } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +Derivation adl_serializer::from_json(const json & json) +{ + return Derivation::fromJSON(json); +} + +void adl_serializer::to_json(json & json, Derivation c) +{ + json = c.toJSON(); +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 08bb7183f..d66bcef2e 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -135,12 +135,11 @@ struct DerivationOutput std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - nlohmann::json toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; + nlohmann::json toJSON(std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -394,11 +393,9 @@ struct Derivation : BasicDerivation { } - nlohmann::json toJSON(const StoreDirConfig & store) const; - static Derivation fromJSON( - const StoreDirConfig & store, - const nlohmann::json & json, - const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON() const; + static Derivation + fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); bool operator==(const Derivation &) const = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. @@ -542,3 +539,5 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName); } // namespace nix + +JSON_IMPL(nix::Derivation) diff --git a/src/libstore/include/nix/store/path.hh b/src/libstore/include/nix/store/path.hh index 784298daa..8124cf580 100644 --- a/src/libstore/include/nix/store/path.hh +++ b/src/libstore/include/nix/store/path.hh @@ -4,6 +4,8 @@ #include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/json-non-null.hh" namespace nix { @@ -87,6 +89,10 @@ typedef std::vector StorePaths; */ constexpr std::string_view drvExtension = ".drv"; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix namespace std { @@ -101,3 +107,5 @@ struct hash }; } // namespace std + +JSON_IMPL(nix::StorePath) diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 516b01571..942f97a88 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,7 @@ +#include + #include "nix/store/store-dir-config.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -75,3 +78,19 @@ StorePath StorePath::random(std::string_view name) } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +StorePath adl_serializer::from_json(const json & json) +{ + return StorePath{getString(json)}; +} + +void adl_serializer::to_json(json & json, StorePath storePath) +{ + json = storePath.to_string(); +} + +} // namespace nlohmann diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 0f797bb20..2d13aba52 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand { auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); - auto drv = Derivation::fromJSON(*store, json); + auto drv = Derivation::fromJSON(json); auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 1a61ccd5c..20e54bba7 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON if (!drvPath.isDerivation()) continue; - jsonRoot[store->printStorePath(drvPath)] = store->readDerivation(drvPath).toJSON(*store); + jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON(); } printJSON(jsonRoot); } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 5cfafbb62..3c24ac2ee 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -62,12 +62,15 @@ builtins.outputOf "hashAlgo": "sha256" } }, - "system": "${system}" + "system": "${system}", + "version": 3 } EOF - drvs[$word]="$(echo "$json" | nix derivation add)" + drvPath=$(echo "$json" | nix derivation add) + storeDir=$(dirname "$drvPath") + drvs[$word]="$(basename "$drvPath")" done - cp "''${drvs[e]}" $out + cp "''${storeDir}/''${drvs[e]}" $out ''; __contentAddressed = true; diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 5dea220fe..9e483d376 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -50,8 +50,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 2bd9b4aaf..dfd5a1412 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -50,4 +50,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") -nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}' +nix derivation show "$hacky" | jq --exit-status '."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'