Skip to content

Commit faddc7d

Browse files
committed
Fix numeric precision loss in JSON parsing
The general purpose `jsonParse` utility was lossy when it comes to numbers containing a decimal point. This affected `json_parse` SQL function, `JSON` SQL type constructor and connectors which use `jsonParse` to canonicalize JSON representation on remote data read (e.g. PostgreSQL).
1 parent 3f6acc4 commit faddc7d

5 files changed

Lines changed: 11 additions & 12 deletions

File tree

core/trino-main/src/test/java/io/trino/type/TestArrayOperators.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -583,7 +583,7 @@ public void testJsonToArraySmoke()
583583

584584
assertTrinoExceptionThrownBy(() -> assertions.expression("CAST(a AS array(INTEGER))")
585585
.binding("a", "JSON '[1234567890123.456]'").evaluate())
586-
.hasMessage("Cannot cast to array(integer). Out of range for integer: 1.234567890123456E12\n[1.234567890123456E12]")
586+
.hasMessage("Cannot cast to array(integer). Out of range for integer: 1.234567890123456E12\n[1234567890123.456]")
587587
.hasErrorCode(INVALID_CAST_ARGUMENT);
588588

589589
assertThat(assertions.expression("CAST(a AS array(DECIMAL(10,5)))")
@@ -1082,11 +1082,10 @@ public void testCastJsonToArrayDecimal()
10821082
.matches("CAST(ARRAY[DECIMAL '12345.88'] AS ARRAY(DECIMAL(7,2)))");
10831083

10841084
// array with large decimal
1085-
// TODO precision loss!
10861085
assertThat(assertions.expression("cast(a as ARRAY(DECIMAL(38,8)))")
10871086
.binding("a", "JSON '[123456789012345678901234567890.12345678]'"))
10881087
.hasType(new ArrayType(createDecimalType(38, 8)))
1089-
.matches("CAST(ARRAY[DECIMAL '123456789012345680000000000000.00000000'] AS ARRAY(DECIMAL(38,8)))");
1088+
.matches("CAST(ARRAY[DECIMAL '123456789012345678901234567890.12345678'] AS ARRAY(DECIMAL(38,8)))");
10901089

10911090
// non-array JSON should fail
10921091
assertTrinoExceptionThrownBy(() -> assertions.expression("cast(a as ARRAY(DECIMAL(10,3)))")

core/trino-main/src/test/java/io/trino/type/TestJsonOperators.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -705,11 +705,10 @@ public void testCastToDecimal()
705705
.hasType(createDecimalType(10, 3))
706706
.isEqualTo(decimal("128.000", createDecimalType(10, 3)));
707707

708-
// TODO precision loss!
709708
assertThat(assertions.expression("cast(a as DECIMAL(38,8))")
710709
.binding("a", "JSON '123456789012345678901234567890.12345678'"))
711710
.hasType(createDecimalType(38, 8))
712-
.isEqualTo(decimal("123456789012345680000000000000.00000000", createDecimalType(38, 8)));
711+
.isEqualTo(decimal("123456789012345678901234567890.12345678", createDecimalType(38, 8)));
713712

714713
assertThat(assertions.expression("cast(a as DECIMAL(38,8))")
715714
.binding("a", "cast(DECIMAL '123456789012345678901234567890.12345678' as JSON)"))
@@ -796,10 +795,10 @@ public void testCastToBoolean()
796795
.binding("a", "JSON '1e-324'"))
797796
.isEqualTo(false);
798797

799-
// overflow
800-
assertTrinoExceptionThrownBy(() -> assertions.expression("cast(a as BOOLEAN)")
801-
.binding("a", "JSON '1e309'").evaluate())
802-
.hasErrorCode(INVALID_CAST_ARGUMENT);
798+
// overflow if parsed as double
799+
assertThat(assertions.expression("cast(a as BOOLEAN)")
800+
.binding("a", "JSON '1e309'"))
801+
.isEqualTo(true);
803802

804803
assertThat(assertions.expression("cast(a as BOOLEAN)")
805804
.binding("a", "JSON 'true'"))

core/trino-main/src/test/java/io/trino/type/TestMapOperators.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ public void testJsonToMap()
683683

684684
assertTrinoExceptionThrownBy(() -> assertions.expression("cast(a as MAP(VARCHAR, INTEGER))")
685685
.binding("a", "JSON '{\"a\": 1234567890123.456}'").evaluate())
686-
.hasMessage("Cannot cast to map(varchar, integer). Out of range for integer: 1.234567890123456E12\n{\"a\":1.234567890123456E12}")
686+
.hasMessage("Cannot cast to map(varchar, integer). Out of range for integer: 1.234567890123456E12\n{\"a\":1234567890123.456}")
687687
.hasErrorCode(INVALID_CAST_ARGUMENT);
688688

689689
assertTrinoExceptionThrownBy(() -> assertions.expression("cast(a as MAP(BIGINT, BIGINT))")

lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/util/JsonTypeUtil.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.StringJoiner;
3737

3838
import static com.fasterxml.jackson.core.JsonFactory.Feature.CANONICALIZE_FIELD_NAMES;
39+
import static com.fasterxml.jackson.databind.DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS;
3940
import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS;
4041
import static com.google.common.base.Preconditions.checkState;
4142
import static io.trino.plugin.base.util.JsonUtils.jsonFactoryBuilder;
@@ -52,6 +53,7 @@ public final class JsonTypeUtil
5253
private static final JsonMapper SORTED_MAPPER = new JsonMapperProvider().get()
5354
.rebuild()
5455
.configure(ORDER_MAP_ENTRIES_BY_KEYS, true)
56+
.configure(USE_BIG_DECIMAL_FOR_FLOATS, true)
5557
.build();
5658

5759
private JsonTypeUtil() {}

lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/util/TestJsonTypeUtil.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,7 @@ void testJsonParseLargeNumber()
7777
assertThat(jsonParse(utf8Slice("12345678901234567890123456789012345678")).toStringUtf8())
7878
.isEqualTo("12345678901234567890123456789012345678");
7979
assertThat(jsonParse(utf8Slice("123456789012345678901234567890.12345678")).toStringUtf8())
80-
// TODO precision loss! Numbers are converted through floating-point instead of being preserved as strings
81-
.isEqualTo("1.2345678901234568E29");
80+
.isEqualTo("123456789012345678901234567890.12345678");
8281
}
8382

8483
@Test

0 commit comments

Comments
 (0)