diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java index 4291634d8f4..8cc7c550f74 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java @@ -551,9 +551,17 @@ public void testTrinoWriteStatsAsJsonEnabled(String type, String inputValue, Dou testWriteStatsAsJsonEnabled(sql -> onTrino().executeQuery(sql), tableName, "delta.default." + tableName, type, inputValue, dataSize, distinctValues, nullsFraction, statsValue); } - @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}, dataProvider = "testDeltaCheckpointWriteStatsAsJson") + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}, dataProvider = "testDeltaCheckpointWriteStatsAsJsonNumericTypes") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) - public void testDatabricksWriteStatsAsJsonEnabled(String type, String inputValue, Double nullsFraction, Object statsValue) + public void testDatabricksWriteStatsAsJsonEnabledNumericTypes(String type, String inputValue, Double nullsFraction, Object statsValue) + { + String tableName = "test_dl_checkpoints_write_stats_as_json_enabled_databricks_" + randomNameSuffix(); + testWriteStatsAsJsonEnabled(sql -> onDelta().executeQuery(sql), tableName, "default." + tableName, type, inputValue, null, null, nullsFraction, statsValue); + } + + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}, dataProvider = "testDeltaCheckpointWriteStatsAsJsonNonNumericTypes") + @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) + public void testDatabricksWriteStatsAsJsonEnabledNonNumericTypes(String type, String inputValue, Double nullsFraction, Object statsValue) { String tableName = "test_dl_checkpoints_write_stats_as_json_enabled_databricks_" + randomNameSuffix(); testWriteStatsAsJsonEnabled(sql -> onDelta().executeQuery(sql), tableName, "default." + tableName, type, inputValue, null, null, nullsFraction, statsValue); @@ -567,9 +575,8 @@ private void testWriteStatsAsJsonEnabled(Consumer sqlExecutor, String ta "USING DELTA " + "LOCATION 's3://%s/databricks-compatibility-test-%1$s' " + "TBLPROPERTIES (" + - " delta.checkpointInterval = 2, " + - " delta.checkpoint.writeStatsAsJson = false, " + - " delta.checkpoint.writeStatsAsStruct = true)", + " delta.checkpoint.writeStatsAsJson = true, " + + " delta.checkpoint.writeStatsAsStruct = false)", tableName, type, bucketName); @@ -580,18 +587,10 @@ private void testWriteStatsAsJsonEnabled(Consumer sqlExecutor, String ta sqlExecutor.accept("INSERT INTO " + qualifiedTableName + " SELECT " + inputValue); sqlExecutor.accept("INSERT INTO " + qualifiedTableName + " SELECT " + inputValue); - // SET TBLPROPERTIES increments checkpoint - onDelta().executeQuery("" + - "ALTER TABLE default." + tableName + " SET TBLPROPERTIES (" + - "'delta.checkpoint.writeStatsAsJson' = true, " + - "'delta.checkpoint.writeStatsAsStruct' = false)"); - - sqlExecutor.accept("INSERT INTO " + qualifiedTableName + " SELECT " + inputValue); - assertThat(onTrino().executeQuery("SHOW STATS FOR delta.default." + tableName)) .containsOnly(ImmutableList.of( row("col", dataSize, distinctValues, nullsFraction, null, statsValue, statsValue), - row(null, null, null, null, 3.0, null, null))); + row(null, null, null, null, 2.0, null, null))); } finally { dropDeltaTableWithRetry("default." + tableName); @@ -622,10 +621,9 @@ public Object[][] testTrinoCheckpointWriteStatsAsJson() } @DataProvider - public Object[][] testDeltaCheckpointWriteStatsAsJson() + public Object[][] testDeltaCheckpointWriteStatsAsJsonNumericTypes() { return new Object[][] { - {"boolean", "true", 0.0, null}, {"integer", "1", 0.0, "1"}, {"tinyint", "2", 0.0, "2"}, {"smallint", "3", 0.0, "3"}, @@ -634,6 +632,14 @@ public Object[][] testDeltaCheckpointWriteStatsAsJson() {"double", "1.0", 0.0, "1.0"}, {"decimal(3,2)", "3.14", 0.0, "3.14"}, {"decimal(30,1)", "12345", 0.0, "12345.0"}, + }; + } + + @DataProvider + public Object[][] testDeltaCheckpointWriteStatsAsJsonNonNumericTypes() + { + return new Object[][] { + {"boolean", "true", 0.0, null}, {"string", "'test'", 0.0, null}, {"binary", "X'65683F'", 0.0, null}, {"date", "date '2021-02-03'", 0.0, "2021-02-03"}, diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java index 40628ef275d..ddd407eaaf7 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCloneTableCompatibility.java @@ -253,10 +253,17 @@ public void testReadFromSchemaChangedShallowCloneTable() @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) - public void testReadFromSchemaChangedDeepCloneTable() + public void testReadFromSchemaChangedDeepCloneTablePartitioned() { // Deep Clone is not supported on Delta-Lake OSS testReadSchemaChangedCloneTable("DEEP", true); + } + + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) + @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) + public void testReadFromSchemaChangedDeepCloneTableNonPartitioned() + { + // Deep Clone is not supported on Delta-Lake OSS testReadSchemaChangedCloneTable("DEEP", false); }