diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 7b5557bd1..190aea76c 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -62,6 +62,7 @@ toml11 = dependency( method : 'cmake', include_type: 'system', ) + deps_other += toml11 config_h = configure_file( diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b4f1df7a8..be449e114 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -7,7 +7,84 @@ namespace nix { -static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) +#ifdef HAVE_TOML11_4 + +/** + * This is what toml11 < 4.0 did when choosing the subsecond precision. + * TOML 1.0.0 spec doesn't define how sub-millisecond ranges should be handled and calls it + * implementation defined behavior. For a lack of a better choice we stick with what older versions + * of toml11 did [1]. + * + * [1]: https://github.com/ToruNiina/toml11/blob/dcfe39a783a94e8d52c885e5883a6fbb21529019/toml/datetime.hpp#L282 + */ +static size_t normalizeSubsecondPrecision(toml::local_time lt) +{ + auto millis = lt.millisecond; + auto micros = lt.microsecond; + auto nanos = lt.nanosecond; + if (millis != 0 || micros != 0 || nanos != 0) { + if (micros != 0 || nanos != 0) { + if (nanos != 0) + return 9; + return 6; + } + return 3; + } + return 0; +} + +/** + * Normalize date/time formats to serialize to the same strings as versions prior to toml11 4.0. + * + * Several things to consider: + * + * 1. Sub-millisecond range is represented the same way as in toml11 versions prior to 4.0. Precisioun is rounded + * towards the next multiple of 3 or capped at 9 digits. + * 2. Seconds must be specified. This may become optional in (yet unreleased) TOML 1.1.0, but 1.0.0 defined local time + * in terms of RFC3339 [1]. + * 3. date-time separator (`t`, `T` or space ` `) is canonicalized to an upper T. This is compliant with RFC3339 + * [1] 5.6: + * > Applications that generate this format SHOULD use upper case letters. + * + * [1]: https://datatracker.ietf.org/doc/html/rfc3339#section-5.6 + */ +static void normalizeDatetimeFormat(toml::value & t) +{ + if (t.is_local_datetime()) { + auto & ldt = t.as_local_datetime(); + t.as_local_datetime_fmt() = { + .delimiter = toml::datetime_delimiter_kind::upper_T, + // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6 + .has_seconds = true, // Mandated by TOML 1.0.0 + .subsecond_precision = normalizeSubsecondPrecision(ldt.time), + }; + return; + } + + if (t.is_offset_datetime()) { + auto & odt = t.as_offset_datetime(); + t.as_offset_datetime_fmt() = { + .delimiter = toml::datetime_delimiter_kind::upper_T, + // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6 + .has_seconds = true, // Mandated by TOML 1.0.0 + .subsecond_precision = normalizeSubsecondPrecision(odt.time), + }; + return; + } + + if (t.is_local_time()) { + auto & lt = t.as_local_time(); + t.as_local_time_fmt() = { + .has_seconds = true, // Mandated by TOML 1.0.0 + .subsecond_precision = normalizeSubsecondPrecision(lt), + }; + return; + } +} + +#endif + +static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Value & val) { auto toml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromTOML"); @@ -16,81 +93,87 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V std::function visit; visit = [&](Value & v, toml::value t) { + switch (t.type()) { + case toml::value_t::table: { + auto table = toml::get(t); - switch(t.type()) - { - case toml::value_t::table: - { - auto table = toml::get(t); + size_t size = 0; + for (auto & i : table) { + (void) i; + size++; + } - size_t size = 0; - for (auto & i : table) { (void) i; size++; } + auto attrs = state.buildBindings(size); - auto attrs = state.buildBindings(size); + for (auto & elem : table) + visit(attrs.alloc(elem.first), elem.second); - for(auto & elem : table) - visit(attrs.alloc(elem.first), elem.second); - - v.mkAttrs(attrs); - } - break;; - case toml::value_t::array: - { - auto array = toml::get>(t); - - auto list = state.buildList(array.size()); - for (const auto & [n, v] : enumerate(list)) - visit(*(v = state.allocValue()), array[n]); - v.mkList(list); - } - break;; - case toml::value_t::boolean: - v.mkBool(toml::get(t)); - break;; - case toml::value_t::integer: - v.mkInt(toml::get(t)); - break;; - case toml::value_t::floating: - v.mkFloat(toml::get(t)); - break;; - case toml::value_t::string: - v.mkString(toml::get(t)); - break;; - case toml::value_t::local_datetime: - case toml::value_t::offset_datetime: - case toml::value_t::local_date: - case toml::value_t::local_time: - { - if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { - auto attrs = state.buildBindings(2); - attrs.alloc("_type").mkString("timestamp"); - std::ostringstream s; - s << t; - attrs.alloc("value").mkString(s.str()); - v.mkAttrs(attrs); - } else { - throw std::runtime_error("Dates and times are not supported"); - } - } - break;; - case toml::value_t::empty: - v.mkNull(); - break;; + v.mkAttrs(attrs); + } break; + case toml::value_t::array: { + auto array = toml::get>(t); + auto list = state.buildList(array.size()); + for (const auto & [n, v] : enumerate(list)) + visit(*(v = state.allocValue()), array[n]); + v.mkList(list); + } break; + case toml::value_t::boolean: + v.mkBool(toml::get(t)); + break; + case toml::value_t::integer: + v.mkInt(toml::get(t)); + break; + case toml::value_t::floating: + v.mkFloat(toml::get(t)); + break; + case toml::value_t::string: + v.mkString(toml::get(t)); + break; + case toml::value_t::local_datetime: + case toml::value_t::offset_datetime: + case toml::value_t::local_date: + case toml::value_t::local_time: { + if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { +#if HAVE_TOML11_4 + normalizeDatetimeFormat(t); +#endif + auto attrs = state.buildBindings(2); + attrs.alloc("_type").mkString("timestamp"); + std::ostringstream s; + s << t; + attrs.alloc("value").mkString(s.str()); + v.mkAttrs(attrs); + } else { + throw std::runtime_error("Dates and times are not supported"); + } + } break; + case toml::value_t::empty: + v.mkNull(); + break; } }; try { - visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); + visit( + val, + toml::parse( + tomlStream, + "fromTOML" /* the "filename" */ +#if HAVE_TOML11_4 + , + toml::spec::v(1, 0, 0) // Be explicit that we are parsing TOML 1.0.0 without extensions +#endif + )); } catch (std::exception & e) { // TODO: toml::syntax_error state.error("while parsing TOML: %s", e.what()).atPos(pos).debugThrow(); } } -static RegisterPrimOp primop_fromTOML({ - .name = "fromTOML", - .args = {"e"}, - .doc = R"( +static RegisterPrimOp primop_fromTOML( + {.name = "fromTOML", + .args = {"e"}, + .doc = R"( Convert a TOML string to a Nix value. For example, ```nix @@ -104,7 +187,6 @@ static RegisterPrimOp primop_fromTOML({ returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. )", - .fun = prim_fromTOML -}); + .fun = prim_fromTOML}); } diff --git a/tests/functional/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp index 08b3c69a6..56e610533 100644 --- a/tests/functional/lang/eval-okay-fromTOML-timestamps.exp +++ b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp @@ -1 +1 @@ -{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } +{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt10 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123456789"; }; ldt11 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123456789"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T07:32:00.100"; }; ldt3 = { _type = "timestamp"; value = "1979-05-27T07:32:00.120"; }; ldt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123"; }; ldt5 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123400"; }; ldt6 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123450"; }; ldt7 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123456"; }; ldt8 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123456700"; }; ldt9 = { _type = "timestamp"; value = "1979-05-27T00:32:00.123456780"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt10 = { _type = "timestamp"; value = "00:32:00.123456789"; }; lt11 = { _type = "timestamp"; value = "00:32:00.123456789"; }; lt2 = { _type = "timestamp"; value = "00:32:00.100"; }; lt3 = { _type = "timestamp"; value = "00:32:00.120"; }; lt4 = { _type = "timestamp"; value = "00:32:00.123"; }; lt5 = { _type = "timestamp"; value = "00:32:00.123400"; }; lt6 = { _type = "timestamp"; value = "00:32:00.123450"; }; lt7 = { _type = "timestamp"; value = "00:32:00.123456"; }; lt8 = { _type = "timestamp"; value = "00:32:00.123456700"; }; lt9 = { _type = "timestamp"; value = "00:32:00.123456780"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt10 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123456Z"; }; odt11 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123456700Z"; }; odt12 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123456780Z"; }; odt13 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123456789Z"; }; odt14 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123456789Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt5 = { _type = "timestamp"; value = "1979-05-27T07:32:00.100Z"; }; odt6 = { _type = "timestamp"; value = "1979-05-27T07:32:00.120Z"; }; odt7 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123Z"; }; odt8 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123400Z"; }; odt9 = { _type = "timestamp"; value = "1979-05-27T07:32:00.123450Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } diff --git a/tests/functional/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix index 74cff9470..d8f3a03e9 100644 --- a/tests/functional/lang/eval-okay-fromTOML-timestamps.nix +++ b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix @@ -55,11 +55,53 @@ builtins.fromTOML '' odt2 = 1979-05-27T00:32:00-07:00 odt3 = 1979-05-27T00:32:00.999999-07:00 odt4 = 1979-05-27 07:32:00Z + # milliseconds + odt5 = 1979-05-27 07:32:00.1Z + odt6 = 1979-05-27 07:32:00.12Z + odt7 = 1979-05-27 07:32:00.123Z + # microseconds + odt8 = 1979-05-27t07:32:00.1234Z + odt9 = 1979-05-27t07:32:00.12345Z + odt10 = 1979-05-27t07:32:00.123456Z + # nanoseconds + odt11 = 1979-05-27 07:32:00.1234567Z + odt12 = 1979-05-27 07:32:00.12345678Z + odt13 = 1979-05-27 07:32:00.123456789Z + # no more precision after nanoseconds + odt14 = 1979-05-27t07:32:00.1234567891Z + ldt1 = 1979-05-27T07:32:00 - ldt2 = 1979-05-27T00:32:00.999999 + # milliseconds + ldt2 = 1979-05-27T07:32:00.1 + ldt3 = 1979-05-27T07:32:00.12 + ldt4 = 1979-05-27T07:32:00.123 + # microseconds + ldt5 = 1979-05-27t00:32:00.1234 + ldt6 = 1979-05-27t00:32:00.12345 + ldt7 = 1979-05-27t00:32:00.123456 + # nanoseconds + ldt8 = 1979-05-27 00:32:00.1234567 + ldt9 = 1979-05-27 00:32:00.12345678 + ldt10 = 1979-05-27 00:32:00.123456789 + # no more precision after nanoseconds + ldt11 = 1979-05-27t00:32:00.1234567891 + ld1 = 1979-05-27 lt1 = 07:32:00 - lt2 = 00:32:00.999999 + # milliseconds + lt2 = 00:32:00.1 + lt3 = 00:32:00.12 + lt4 = 00:32:00.123 + # microseconds + lt5 = 00:32:00.1234 + lt6 = 00:32:00.12345 + lt7 = 00:32:00.123456 + # nanoseconds + lt8 = 00:32:00.1234567 + lt9 = 00:32:00.12345678 + lt10 = 00:32:00.123456789 + # no more precision after nanoseconds + lt11 = 00:32:00.1234567891 arr1 = [ 1, 2, 3 ] arr2 = [ "red", "yellow", "green" ]