Merge pull request #14057 from obsidiansystems/derived-path-json

Modernize and test derived path JSON

src/libcmd/built-path.cc

@@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons
 
 nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const SingleBuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
+    return std::visit(
+        overloaded{
+            [&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); },
+            [&](const BuiltPath::Built & b) { return b.toJSON(store); },
+        },
+        raw());
 }
 
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const

src/libstore-tests/data/derived-path/multi_built_built.json

@@ -0,0 +1,10 @@
+{
+  "drvPath": {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+    "output": "bar"
+  },
+  "outputs": [
+    "baz",
+    "quux"
+  ]
+}

src/libstore-tests/data/derived-path/multi_built_built_wildcard.json

@@ -0,0 +1,9 @@
+{
+  "drvPath": {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+    "output": "bar"
+  },
+  "outputs": [
+    "*"
+  ]
+}

src/libstore-tests/data/derived-path/multi_opaque.json

@@ -0,0 +1 @@
+"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"

src/libstore-tests/data/derived-path/mutli_built.json

@@ -0,0 +1,7 @@
+{
+  "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  "outputs": [
+    "bar",
+    "baz"
+  ]
+}

src/libstore-tests/data/derived-path/single_built.json

@@ -0,0 +1,4 @@
+{
+  "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  "output": "bar"
+}

src/libstore-tests/data/derived-path/single_built_built.json

@@ -0,0 +1,7 @@
+{
+  "drvPath": {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+    "output": "bar"
+  },
+  "output": "baz"
+}

src/libstore-tests/data/derived-path/single_opaque.json

@@ -0,0 +1 @@
+"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"

src/libstore-tests/derived-path.cc

@@ -3,13 +3,23 @@
 #include <gtest/gtest.h>
 #include <rapidcheck/gtest.h>
 
+#include "nix/util/tests/characterization.hh"
 #include "nix/store/tests/derived-path.hh"
 #include "nix/store/tests/libstore.hh"
 
 namespace nix {
 
-class DerivedPathTest : public LibStoreTest
-{};
+class DerivedPathTest : public CharacterizationTest, public LibStoreTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "derived-path";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
 
 /**
  * Round trip (string <-> data structure) test for
@@ -107,4 +117,90 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o))
 
 #endif
 
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+using nlohmann::json;
+
+#define TEST_JSON(TYPE, NAME, VAL)                                                                    \
+    static const TYPE NAME = VAL;                                                                     \
+                                                                                                      \
+    TEST_F(DerivedPathTest, NAME##_from_json)                                                         \
+    {                                                                                                 \
+        readTest(#NAME ".json", [&](const auto & encoded_) {                                          \
+            auto encoded = json::parse(encoded_);                                                     \
+            TYPE got = static_cast<TYPE>(encoded);                                                    \
+            ASSERT_EQ(got, NAME);                                                                     \
+        });                                                                                           \
+    }                                                                                                 \
+                                                                                                      \
+    TEST_F(DerivedPathTest, NAME##_to_json)                                                           \
+    {                                                                                                 \
+        writeTest(                                                                                    \
+            #NAME ".json",                                                                            \
+            [&]() -> json { return static_cast<json>(NAME); },                                        \
+            [](const auto & file) { return json::parse(readFile(file)); },                            \
+            [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
+    }
+
+TEST_JSON(
+    SingleDerivedPath, single_opaque, SingleDerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}});
+
+TEST_JSON(
+    SingleDerivedPath,
+    single_built,
+    (SingleDerivedPath::Built{
+        .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{
+            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}),
+        .output = "bar",
+    }));
+
+TEST_JSON(
+    SingleDerivedPath,
+    single_built_built,
+    (SingleDerivedPath::Built{
+        .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
+            .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{
+                StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}),
+            .output = "bar",
+        }),
+        .output = "baz",
+    }));
+
+TEST_JSON(DerivedPath, multi_opaque, DerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}});
+
+TEST_JSON(
+    DerivedPath,
+    mutli_built,
+    (DerivedPath::Built{
+        .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{
+            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}),
+        .outputs = OutputsSpec::Names{"bar", "baz"},
+    }));
+
+TEST_JSON(
+    DerivedPath,
+    multi_built_built,
+    (DerivedPath::Built{
+        .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
+            .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{
+                StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}),
+            .output = "bar",
+        }),
+        .outputs = OutputsSpec::Names{"baz", "quux"},
+    }));
+
+TEST_JSON(
+    DerivedPath,
+    multi_built_built_wildcard,
+    (DerivedPath::Built{
+        .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
+            .drvPath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{
+                StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}),
+            .output = "bar",
+        }),
+        .outputs = OutputsSpec::All{},
+    }));
+
 } // namespace nix

src/libstore/derived-path.cc

@@ -2,8 +2,7 @@
 #include "nix/store/derivations.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/comparator.hh"
-
-#include <nlohmann/json.hpp>
+#include "nix/util/json-utils.hh"
 
 #include <optional>
 
@@ -19,59 +18,6 @@ GENERATE_CMP_EXT(, std::strong_ordering, SingleDerivedPathBuilt, *me->drvPath, m
 GENERATE_EQUAL(, DerivedPathBuilt ::, DerivedPathBuilt, *me->drvPath, me->outputs);
 GENERATE_ONE_CMP(, bool, DerivedPathBuilt ::, <, DerivedPathBuilt, *me->drvPath, me->outputs);
 
-nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
-{
-    return store.printStorePath(path);
-}
-
-nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const
-{
-    nlohmann::json res;
-    res["drvPath"] = drvPath->toJSON(store);
-    // Fallback for the input-addressed derivation case: We expect to always be
-    // able to print the output paths, so let’s do it
-    // FIXME try-resolve on drvPath
-    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
-    res["output"] = output;
-    auto outputPathIter = outputMap.find(output);
-    if (outputPathIter == outputMap.end())
-        res["outputPath"] = nullptr;
-    else if (std::optional p = outputPathIter->second)
-        res["outputPath"] = store.printStorePath(*p);
-    else
-        res["outputPath"] = nullptr;
-    return res;
-}
-
-nlohmann::json DerivedPath::Built::toJSON(Store & store) const
-{
-    nlohmann::json res;
-    res["drvPath"] = drvPath->toJSON(store);
-    // Fallback for the input-addressed derivation case: We expect to always be
-    // able to print the output paths, so let’s do it
-    // FIXME try-resolve on drvPath
-    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
-    for (const auto & [output, outputPathOpt] : outputMap) {
-        if (!outputs.contains(output))
-            continue;
-        if (outputPathOpt)
-            res["outputs"][output] = store.printStorePath(*outputPathOpt);
-        else
-            res["outputs"][output] = nullptr;
-    }
-    return res;
-}
-
-nlohmann::json SingleDerivedPath::toJSON(Store & store) const
-{
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
-}
-
-nlohmann::json DerivedPath::toJSON(Store & store) const
-{
-    return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw());
-}
-
 std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
@@ -273,3 +219,77 @@ const StorePath & DerivedPath::getBaseStorePath() const
 }
 
 } // namespace nix
+
+namespace nlohmann {
+
+void adl_serializer<SingleDerivedPath::Opaque>::to_json(json & json, const SingleDerivedPath::Opaque & o)
+{
+    json = o.path;
+}
+
+SingleDerivedPath::Opaque adl_serializer<SingleDerivedPath::Opaque>::from_json(const json & json)
+{
+    return SingleDerivedPath::Opaque{json};
+}
+
+void adl_serializer<SingleDerivedPath::Built>::to_json(json & json, const SingleDerivedPath::Built & sdpb)
+{
+    json = {
+        {"drvPath", *sdpb.drvPath},
+        {"output", sdpb.output},
+    };
+}
+
+void adl_serializer<DerivedPath::Built>::to_json(json & json, const DerivedPath::Built & dbp)
+{
+    json = {
+        {"drvPath", *dbp.drvPath},
+        {"outputs", dbp.outputs},
+    };
+}
+
+SingleDerivedPath::Built adl_serializer<SingleDerivedPath::Built>::from_json(const json & json0)
+{
+    auto & json = getObject(json0);
+    return {
+        .drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
+        .output = getString(valueAt(json, "output")),
+    };
+}
+
+DerivedPath::Built adl_serializer<DerivedPath::Built>::from_json(const json & json0)
+{
+    auto & json = getObject(json0);
+    return {
+        .drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
+        .outputs = adl_serializer<OutputsSpec>::from_json(valueAt(json, "outputs")),
+    };
+}
+
+void adl_serializer<SingleDerivedPath>::to_json(json & json, const SingleDerivedPath & sdp)
+{
+    std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw());
+}
+
+void adl_serializer<DerivedPath>::to_json(json & json, const DerivedPath & sdp)
+{
+    std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw());
+}
+
+SingleDerivedPath adl_serializer<SingleDerivedPath>::from_json(const json & json)
+{
+    if (json.is_string())
+        return static_cast<SingleDerivedPath::Opaque>(json);
+    else
+        return static_cast<SingleDerivedPath::Built>(json);
+}
+
+DerivedPath adl_serializer<DerivedPath>::from_json(const json & json)
+{
+    if (json.is_string())
+        return static_cast<DerivedPath::Opaque>(json);
+    else
+        return static_cast<DerivedPath::Built>(json);
+}
+
+} // namespace nlohmann

src/libstore/include/nix/store/derived-path.hh

@@ -5,6 +5,7 @@
 #include "nix/store/outputs-spec.hh"
 #include "nix/util/configuration.hh"
 #include "nix/util/ref.hh"
+#include "nix/util/json-impls.hh"
 
 #include <variant>
 
@@ -14,9 +15,6 @@ namespace nix {
 
 struct StoreDirConfig;
 
-// TODO stop needing this, `toJSON` below should be pure
-class Store;
-
 /**
  * An opaque derived path.
  *
@@ -30,7 +28,6 @@ struct DerivedPathOpaque
 
     std::string to_string(const StoreDirConfig & store) const;
     static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
-    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     bool operator==(const DerivedPathOpaque &) const = default;
     auto operator<=>(const DerivedPathOpaque &) const = default;
@@ -80,7 +77,6 @@ struct SingleDerivedPathBuilt
         ref<const SingleDerivedPath> drvPath,
         OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
-    nlohmann::json toJSON(Store & store) const;
 
     bool operator==(const SingleDerivedPathBuilt &) const noexcept;
     std::strong_ordering operator<=>(const SingleDerivedPathBuilt &) const noexcept;
@@ -153,7 +149,6 @@ struct SingleDerivedPath : _SingleDerivedPathRaw
         const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
-    nlohmann::json toJSON(Store & store) const;
 };
 
 static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath)
@@ -208,7 +203,6 @@ struct DerivedPathBuilt
         ref<const SingleDerivedPath>,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
-    nlohmann::json toJSON(Store & store) const;
 
     bool operator==(const DerivedPathBuilt &) const noexcept;
     // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet.
@@ -287,8 +281,6 @@ struct DerivedPath : _DerivedPathRaw
      * Convert a `SingleDerivedPath` to a `DerivedPath`.
      */
     static DerivedPath fromSingle(const SingleDerivedPath &);
-
-    nlohmann::json toJSON(Store & store) const;
 };
 
 typedef std::vector<DerivedPath> DerivedPaths;
@@ -305,3 +297,9 @@ typedef std::vector<DerivedPath> DerivedPaths;
 void drvRequireExperiment(
     const SingleDerivedPath & drv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 } // namespace nix
+
+JSON_IMPL(nix::SingleDerivedPath::Opaque)
+JSON_IMPL(nix::SingleDerivedPath::Built)
+JSON_IMPL(nix::SingleDerivedPath)
+JSON_IMPL(nix::DerivedPath::Built)
+JSON_IMPL(nix::DerivedPath)

src/nix/build.cc

@@ -8,11 +8,79 @@
 
 using namespace nix;
 
+/* This serialization code is diferent from the canonical (single)
+   derived path serialization because:
+
+   - It looks up output paths where possible
+
+   - It includes the store dir in store paths
+
+   We might want to replace it with the canonical format at some point,
+   but that would be a breaking change (to a still-experimental but
+   widely-used command, so that isn't being done at this time just yet.
+ */
+
+static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Opaque & o)
+{
+    return store.printStorePath(o.path);
+}
+
+static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp);
+static nlohmann::json toJSON(Store & store, const DerivedPath & dp);
+
+static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Built & sdpb)
+{
+    nlohmann::json res;
+    res["drvPath"] = toJSON(store, *sdpb.drvPath);
+    // Fallback for the input-addressed derivation case: We expect to always be
+    // able to print the output paths, so let’s do it
+    // FIXME try-resolve on drvPath
+    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *sdpb.drvPath));
+    res["output"] = sdpb.output;
+    auto outputPathIter = outputMap.find(sdpb.output);
+    if (outputPathIter == outputMap.end())
+        res["outputPath"] = nullptr;
+    else if (std::optional p = outputPathIter->second)
+        res["outputPath"] = store.printStorePath(*p);
+    else
+        res["outputPath"] = nullptr;
+    return res;
+}
+
+static nlohmann::json toJSON(Store & store, const DerivedPath::Built & dpb)
+{
+    nlohmann::json res;
+    res["drvPath"] = toJSON(store, *dpb.drvPath);
+    // Fallback for the input-addressed derivation case: We expect to always be
+    // able to print the output paths, so let’s do it
+    // FIXME try-resolve on drvPath
+    const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *dpb.drvPath));
+    for (const auto & [output, outputPathOpt] : outputMap) {
+        if (!dpb.outputs.contains(output))
+            continue;
+        if (outputPathOpt)
+            res["outputs"][output] = store.printStorePath(*outputPathOpt);
+        else
+            res["outputs"][output] = nullptr;
+    }
+    return res;
+}
+
+static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp)
+{
+    return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, sdp.raw());
+}
+
+static nlohmann::json toJSON(Store & store, const DerivedPath & dp)
+{
+    return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, dp.raw());
+}
+
 static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store)
 {
     auto res = nlohmann::json::array();
     for (auto & t : paths) {
-        res.push_back(t.toJSON(store));
+        res.push_back(toJSON(store, t));
     }
     return res;
 }