From d23e59bb6bb5b429bc8ca0e303f03cbb385130c3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Feb 2025 00:52:05 -0500 Subject: [PATCH] Modernize and test derived path JSON Old code is now just used for `nix build` --- there is no CLI breaking change. Test the new format, too. The new format is not currently used, but will be used going forward, for example in the C API. Progress on #13570 --- src/libcmd/built-path.cc | 14 +- .../data/derived-path/multi_built_built.json | 10 ++ .../multi_built_built_wildcard.json | 9 ++ .../data/derived-path/multi_opaque.json | 1 + .../data/derived-path/mutli_built.json | 7 + .../data/derived-path/single_built.json | 4 + .../data/derived-path/single_built_built.json | 7 + .../data/derived-path/single_opaque.json | 1 + src/libstore-tests/derived-path.cc | 100 +++++++++++++- src/libstore/derived-path.cc | 130 ++++++++++-------- .../include/nix/store/derived-path.hh | 16 +-- src/nix/build.cc | 70 +++++++++- 12 files changed, 300 insertions(+), 69 deletions(-) create mode 100644 src/libstore-tests/data/derived-path/multi_built_built.json create mode 100644 src/libstore-tests/data/derived-path/multi_built_built_wildcard.json create mode 100644 src/libstore-tests/data/derived-path/multi_opaque.json create mode 100644 src/libstore-tests/data/derived-path/mutli_built.json create mode 100644 src/libstore-tests/data/derived-path/single_built.json create mode 100644 src/libstore-tests/data/derived-path/single_built_built.json create mode 100644 src/libstore-tests/data/derived-path/single_opaque.json diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 80d97dc3e..4d76dd6da 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); + return std::visit( + overloaded{ + [&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); }, + [&](const SingleBuiltPath::Built & b) { return b.toJSON(store); }, + }, + raw()); } nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); + return std::visit( + overloaded{ + [&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); }, + [&](const BuiltPath::Built & b) { return b.toJSON(store); }, + }, + raw()); } RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const diff --git a/src/libstore-tests/data/derived-path/multi_built_built.json b/src/libstore-tests/data/derived-path/multi_built_built.json new file mode 100644 index 000000000..561d04850 --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_built_built.json @@ -0,0 +1,10 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "outputs": [ + "baz", + "quux" + ] +} diff --git a/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json b/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json new file mode 100644 index 000000000..da1f9d996 --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json @@ -0,0 +1,9 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "outputs": [ + "*" + ] +} diff --git a/src/libstore-tests/data/derived-path/multi_opaque.json b/src/libstore-tests/data/derived-path/multi_opaque.json new file mode 100644 index 000000000..9bedb882b --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_opaque.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/data/derived-path/mutli_built.json b/src/libstore-tests/data/derived-path/mutli_built.json new file mode 100644 index 000000000..d7bcff53d --- /dev/null +++ b/src/libstore-tests/data/derived-path/mutli_built.json @@ -0,0 +1,7 @@ +{ + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputs": [ + "bar", + "baz" + ] +} diff --git a/src/libstore-tests/data/derived-path/single_built.json b/src/libstore-tests/data/derived-path/single_built.json new file mode 100644 index 000000000..64110a364 --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_built.json @@ -0,0 +1,4 @@ +{ + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" +} diff --git a/src/libstore-tests/data/derived-path/single_built_built.json b/src/libstore-tests/data/derived-path/single_built_built.json new file mode 100644 index 000000000..66faa668c --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_built_built.json @@ -0,0 +1,7 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "output": "baz" +} diff --git a/src/libstore-tests/data/derived-path/single_opaque.json b/src/libstore-tests/data/derived-path/single_opaque.json new file mode 100644 index 000000000..9bedb882b --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_opaque.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index c7d2c5817..6e7648f25 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -3,13 +3,23 @@ #include #include +#include "nix/util/tests/characterization.hh" #include "nix/store/tests/derived-path.hh" #include "nix/store/tests/libstore.hh" namespace nix { -class DerivedPathTest : public LibStoreTest -{}; +class DerivedPathTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derived-path"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; /** * Round trip (string <-> data structure) test for @@ -107,4 +117,90 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) #endif +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +#define TEST_JSON(TYPE, NAME, VAL) \ + static const TYPE NAME = VAL; \ + \ + TEST_F(DerivedPathTest, NAME##_from_json) \ + { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + TYPE got = static_cast(encoded); \ + ASSERT_EQ(got, NAME); \ + }); \ + } \ + \ + TEST_F(DerivedPathTest, NAME##_to_json) \ + { \ + writeTest( \ + #NAME ".json", \ + [&]() -> json { return static_cast(NAME); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +TEST_JSON( + SingleDerivedPath, single_opaque, SingleDerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}); + +TEST_JSON( + SingleDerivedPath, + single_built, + (SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + })); + +TEST_JSON( + SingleDerivedPath, + single_built_built, + (SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .output = "baz", + })); + +TEST_JSON(DerivedPath, multi_opaque, DerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}); + +TEST_JSON( + DerivedPath, + mutli_built, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .outputs = OutputsSpec::Names{"bar", "baz"}, + })); + +TEST_JSON( + DerivedPath, + multi_built_built, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .outputs = OutputsSpec::Names{"baz", "quux"}, + })); + +TEST_JSON( + DerivedPath, + multi_built_built_wildcard, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .outputs = OutputsSpec::All{}, + })); + } // namespace nix diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 1fee1ae75..2cf720b82 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -2,8 +2,7 @@ #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/util/comparator.hh" - -#include +#include "nix/util/json-utils.hh" #include @@ -19,59 +18,6 @@ GENERATE_CMP_EXT(, std::strong_ordering, SingleDerivedPathBuilt, *me->drvPath, m GENERATE_EQUAL(, DerivedPathBuilt ::, DerivedPathBuilt, *me->drvPath, me->outputs); GENERATE_ONE_CMP(, bool, DerivedPathBuilt ::, <, DerivedPathBuilt, *me->drvPath, me->outputs); -nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const -{ - return store.printStorePath(path); -} - -nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const -{ - nlohmann::json res; - res["drvPath"] = drvPath->toJSON(store); - // Fallback for the input-addressed derivation case: We expect to always be - // able to print the output paths, so let’s do it - // FIXME try-resolve on drvPath - const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); - res["output"] = output; - auto outputPathIter = outputMap.find(output); - if (outputPathIter == outputMap.end()) - res["outputPath"] = nullptr; - else if (std::optional p = outputPathIter->second) - res["outputPath"] = store.printStorePath(*p); - else - res["outputPath"] = nullptr; - return res; -} - -nlohmann::json DerivedPath::Built::toJSON(Store & store) const -{ - nlohmann::json res; - res["drvPath"] = drvPath->toJSON(store); - // Fallback for the input-addressed derivation case: We expect to always be - // able to print the output paths, so let’s do it - // FIXME try-resolve on drvPath - const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); - for (const auto & [output, outputPathOpt] : outputMap) { - if (!outputs.contains(output)) - continue; - if (outputPathOpt) - res["outputs"][output] = store.printStorePath(*outputPathOpt); - else - res["outputs"][output] = nullptr; - } - return res; -} - -nlohmann::json SingleDerivedPath::toJSON(Store & store) const -{ - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); -} - -nlohmann::json DerivedPath::toJSON(Store & store) const -{ - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); -} - std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); @@ -273,3 +219,77 @@ const StorePath & DerivedPath::getBaseStorePath() const } } // namespace nix + +namespace nlohmann { + +void adl_serializer::to_json(json & json, const SingleDerivedPath::Opaque & o) +{ + json = o.path; +} + +SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) +{ + return SingleDerivedPath::Opaque{json}; +} + +void adl_serializer::to_json(json & json, const SingleDerivedPath::Built & sdpb) +{ + json = { + {"drvPath", *sdpb.drvPath}, + {"output", sdpb.output}, + }; +} + +void adl_serializer::to_json(json & json, const DerivedPath::Built & dbp) +{ + json = { + {"drvPath", *dbp.drvPath}, + {"outputs", dbp.outputs}, + }; +} + +SingleDerivedPath::Built adl_serializer::from_json(const json & json0) +{ + auto & json = getObject(json0); + return { + .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .output = getString(valueAt(json, "output")), + }; +} + +DerivedPath::Built adl_serializer::from_json(const json & json0) +{ + auto & json = getObject(json0); + return { + .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .outputs = adl_serializer::from_json(valueAt(json, "outputs")), + }; +} + +void adl_serializer::to_json(json & json, const SingleDerivedPath & sdp) +{ + std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); +} + +void adl_serializer::to_json(json & json, const DerivedPath & sdp) +{ + std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); +} + +SingleDerivedPath adl_serializer::from_json(const json & json) +{ + if (json.is_string()) + return static_cast(json); + else + return static_cast(json); +} + +DerivedPath adl_serializer::from_json(const json & json) +{ + if (json.is_string()) + return static_cast(json); + else + return static_cast(json); +} + +} // namespace nlohmann diff --git a/src/libstore/include/nix/store/derived-path.hh b/src/libstore/include/nix/store/derived-path.hh index bc89b012e..47b29b2d6 100644 --- a/src/libstore/include/nix/store/derived-path.hh +++ b/src/libstore/include/nix/store/derived-path.hh @@ -5,6 +5,7 @@ #include "nix/store/outputs-spec.hh" #include "nix/util/configuration.hh" #include "nix/util/ref.hh" +#include "nix/util/json-impls.hh" #include @@ -14,9 +15,6 @@ namespace nix { struct StoreDirConfig; -// TODO stop needing this, `toJSON` below should be pure -class Store; - /** * An opaque derived path. * @@ -30,7 +28,6 @@ struct DerivedPathOpaque std::string to_string(const StoreDirConfig & store) const; static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); - nlohmann::json toJSON(const StoreDirConfig & store) const; bool operator==(const DerivedPathOpaque &) const = default; auto operator<=>(const DerivedPathOpaque &) const = default; @@ -80,7 +77,6 @@ struct SingleDerivedPathBuilt ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; bool operator==(const SingleDerivedPathBuilt &) const noexcept; std::strong_ordering operator<=>(const SingleDerivedPathBuilt &) const noexcept; @@ -153,7 +149,6 @@ struct SingleDerivedPath : _SingleDerivedPathRaw const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; }; static inline ref makeConstantStorePathRef(StorePath drvPath) @@ -208,7 +203,6 @@ struct DerivedPathBuilt ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; bool operator==(const DerivedPathBuilt &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. @@ -287,8 +281,6 @@ struct DerivedPath : _DerivedPathRaw * Convert a `SingleDerivedPath` to a `DerivedPath`. */ static DerivedPath fromSingle(const SingleDerivedPath &); - - nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; @@ -305,3 +297,9 @@ typedef std::vector DerivedPaths; void drvRequireExperiment( const SingleDerivedPath & drv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } // namespace nix + +JSON_IMPL(nix::SingleDerivedPath::Opaque) +JSON_IMPL(nix::SingleDerivedPath::Built) +JSON_IMPL(nix::SingleDerivedPath) +JSON_IMPL(nix::DerivedPath::Built) +JSON_IMPL(nix::DerivedPath) diff --git a/src/nix/build.cc b/src/nix/build.cc index ea05251ad..2d4f426a4 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -8,11 +8,79 @@ using namespace nix; +/* This serialization code is diferent from the canonical (single) + derived path serialization because: + + - It looks up output paths where possible + + - It includes the store dir in store paths + + We might want to replace it with the canonical format at some point, + but that would be a breaking change (to a still-experimental but + widely-used command, so that isn't being done at this time just yet. + */ + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Opaque & o) +{ + return store.printStorePath(o.path); +} + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp); +static nlohmann::json toJSON(Store & store, const DerivedPath & dp); + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Built & sdpb) +{ + nlohmann::json res; + res["drvPath"] = toJSON(store, *sdpb.drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *sdpb.drvPath)); + res["output"] = sdpb.output; + auto outputPathIter = outputMap.find(sdpb.output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; + return res; +} + +static nlohmann::json toJSON(Store & store, const DerivedPath::Built & dpb) +{ + nlohmann::json res; + res["drvPath"] = toJSON(store, *dpb.drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *dpb.drvPath)); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!dpb.outputs.contains(output)) + continue; + if (outputPathOpt) + res["outputs"][output] = store.printStorePath(*outputPathOpt); + else + res["outputs"][output] = nullptr; + } + return res; +} + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp) +{ + return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, sdp.raw()); +} + +static nlohmann::json toJSON(Store & store, const DerivedPath & dp) +{ + return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, dp.raw()); +} + static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - res.push_back(t.toJSON(store)); + res.push_back(toJSON(store, t)); } return res; }