diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56aad787b544..6bbcf13ad15f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -885,6 +885,7 @@ jobs: - suite-delta-lake-databricks164 - suite-ranger - suite-gcs + - suite-hive4 - suite-clients - suite-functions - suite-tpch diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java index 0aa747d0eafb..e3e9948fee64 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java @@ -72,6 +72,7 @@ import static io.trino.orc.OrcWriterStats.FlushReason.DICTIONARY_FULL; import static io.trino.orc.OrcWriterStats.FlushReason.MAX_BYTES; import static io.trino.orc.OrcWriterStats.FlushReason.MAX_ROWS; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; import static io.trino.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; import static io.trino.orc.metadata.OrcColumnId.ROOT_COLUMN; import static io.trino.orc.metadata.PostScript.MAGIC; @@ -529,7 +530,8 @@ private List bufferFileFooter() orcTypes, fileStats, userMetadata, - Optional.empty()); // writer id will be set by MetadataWriter + Optional.empty(), // writer id will be set by MetadataWriter + PROLEPTIC_GREGORIAN); closedStripes.clear(); closedStripesRetainedBytes = 0; diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java new file mode 100644 index 000000000000..468e241497f0 --- /dev/null +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.orc.metadata; + +public enum CalendarKind +{ + UNKNOWN_CALENDAR, + JULIAN_GREGORIAN, + PROLEPTIC_GREGORIAN +} diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java index b1d2daf22ee3..2ba27f0cfbab 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java @@ -37,6 +37,7 @@ public class Footer private final Optional> fileStats; private final Map userMetadata; private final Optional writerId; + private final CalendarKind calendar; public Footer( long numberOfRows, @@ -45,7 +46,8 @@ public Footer( ColumnMetadata types, Optional> fileStats, Map userMetadata, - Optional writerId) + Optional writerId, + CalendarKind calendar) { this.numberOfRows = numberOfRows; rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1")); @@ -56,6 +58,7 @@ public Footer( requireNonNull(userMetadata, "userMetadata is null"); this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slice::copy)); this.writerId = requireNonNull(writerId, "writerId is null"); + this.calendar = requireNonNull(calendar, "calendar is null"); } public long getNumberOfRows() @@ -93,6 +96,11 @@ public Optional getWriterId() return writerId; } + public CalendarKind getCalendar() + { + return calendar; + } + @Override public String toString() { @@ -104,6 +112,7 @@ public String toString() .add("columnStatistics", fileStats) .add("userMetadata", userMetadata.keySet()) .add("writerId", writerId) + .add("calendar", calendar) .toString(); } } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java index d32c36b90ae4..f0f5e4c4e50f 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java @@ -56,6 +56,9 @@ import static io.airlift.slice.SliceUtf8.lengthOfCodePoint; import static io.airlift.slice.SliceUtf8.tryGetCodePointAt; import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.trino.orc.metadata.CalendarKind.JULIAN_GREGORIAN; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; +import static io.trino.orc.metadata.CalendarKind.UNKNOWN_CALENDAR; import static io.trino.orc.metadata.CompressionKind.LZ4; import static io.trino.orc.metadata.CompressionKind.NONE; import static io.trino.orc.metadata.CompressionKind.SNAPPY; @@ -150,7 +153,8 @@ public Footer readFooter(HiveWriterVersion hiveWriterVersion, InputStream inputS toType(footer.getTypesList()), toColumnStatistics(hiveWriterVersion, footer.getStatisticsList(), false), toUserMetadata(footer.getMetadataList()), - Optional.of(footer.getWriter())); + Optional.of(footer.getWriter()), + toTrinoOrcCalendarKind(footer.getCalendar())); } private static List toStripeInformation(List types) @@ -409,6 +413,16 @@ private static BinaryStatistics toBinaryStatistics(OrcProto.BinaryStatistics bin return new BinaryStatistics(binaryStatistics.getSum()); } + private static CalendarKind toTrinoOrcCalendarKind(OrcProto.CalendarKind calendarKind) + { + return switch (calendarKind) { + case null -> UNKNOWN_CALENDAR; + case OrcProto.CalendarKind.UNKNOWN_CALENDAR -> UNKNOWN_CALENDAR; + case OrcProto.CalendarKind.JULIAN_GREGORIAN -> JULIAN_GREGORIAN; + case OrcProto.CalendarKind.PROLEPTIC_GREGORIAN -> PROLEPTIC_GREGORIAN; + }; + } + private static Slice byteStringToSlice(ByteString value) { return Slices.wrappedBuffer(value.toByteArray()); diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java index 5f62c3cc300e..d893e5c3bfde 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java @@ -141,7 +141,8 @@ public int writeFooter(SliceOutput output, Footer footer) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) - .collect(toList())); + .collect(toList())) + .setCalendar(toOrcCalendarKind(footer.getCalendar())); setWriter(builder); @@ -361,6 +362,15 @@ private static OrcProto.Stream.Kind toStreamKind(StreamKind streamKind) throw new IllegalArgumentException("Unsupported stream kind: " + streamKind); } + private static OrcProto.CalendarKind toOrcCalendarKind(CalendarKind calendarKind) + { + return switch (calendarKind) { + case UNKNOWN_CALENDAR -> OrcProto.CalendarKind.UNKNOWN_CALENDAR; + case JULIAN_GREGORIAN -> OrcProto.CalendarKind.JULIAN_GREGORIAN; + case PROLEPTIC_GREGORIAN -> OrcProto.CalendarKind.PROLEPTIC_GREGORIAN; + }; + } + private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { return OrcProto.ColumnEncoding.newBuilder() diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java index 0cad039902a8..79fd5fb59822 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java @@ -21,6 +21,7 @@ import io.airlift.units.DataSize; import io.trino.filesystem.local.LocalOutputFile; import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; +import io.trino.orc.metadata.CompressionKind; import io.trino.orc.metadata.Footer; import io.trino.orc.metadata.OrcMetadataReader; import io.trino.orc.metadata.OrcType; @@ -32,23 +33,33 @@ import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.block.VariableWidthBlockBuilder; +import io.trino.spi.type.SqlDate; +import io.trino.spi.type.SqlTimestamp; import io.trino.spi.type.Type; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.InputStream; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; import java.util.List; import java.util.Optional; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.orc.OrcTester.READER_OPTIONS; import static io.trino.orc.StripeReader.isIndexStream; import static io.trino.orc.TestingOrcPredicate.ORC_ROW_GROUP_SIZE; import static io.trino.orc.TestingOrcPredicate.ORC_STRIPE_SIZE; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; import static io.trino.orc.metadata.CompressionKind.NONE; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.testing.DateTimeTestingUtils.sqlDateOf; +import static io.trino.testing.DateTimeTestingUtils.sqlTimestampOf; import static java.lang.Math.toIntExact; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -78,6 +89,38 @@ public void testWriteHugeChunk() testWriteOutput(columnNameBuilder.build(), data); } + @Test + public void testCalendarEntryInFooter() + { + List strings = ImmutableList.of("aaa1", "qwerty", "asdf", "zxcvb", "1234"); + assertFooterHasProlepticGregorianCalendar(VARCHAR, strings); + + List dates = ImmutableList.of("2020-01-01", "2021-02-02", "2022-03-03", "2023-04-04", "2024-05-05").stream() + .map(text -> sqlDateOf(LocalDate.parse(text))) + .collect(toImmutableList()); + assertFooterHasProlepticGregorianCalendar(DATE, dates); + + List timestamps = ImmutableList.of("2023-04-11T05:16:12.123", "2021-04-11T05:16:12.123", "1999-04-11T05:16:12.123").stream() + .map(text -> sqlTimestampOf(TIMESTAMP_MILLIS.getPrecision(), LocalDateTime.parse(text))) + .collect(toImmutableList()); + assertFooterHasProlepticGregorianCalendar(TIMESTAMP_MILLIS, timestamps); + } + + private static void assertFooterHasProlepticGregorianCalendar(Type type, List values) + { + try (TempFile tempFile = new TempFile()) { + OrcTester.writeOrcColumnTrino(tempFile.getFile(), CompressionKind.NONE, type, values.iterator(), new OrcWriterStats()); + + OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), READER_OPTIONS); + + assertThat(OrcReader.createOrcReader(orcDataSource, READER_OPTIONS).orElseThrow(() -> new RuntimeException("File is empty")).getFooter().getCalendar()) + .isEqualTo(PROLEPTIC_GREGORIAN); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + private void testWriteOutput(List columnNames, String[] data) throws IOException { diff --git a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java index cb81b05109b3..bd60d0d09e70 100644 --- a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java @@ -50,6 +50,7 @@ public final class TestGroups public static final String HDFS_IMPERSONATION = "hdfs_impersonation"; public static final String HDFS_NO_IMPERSONATION = "hdfs_no_impersonation"; public static final String HIVE_GCS = "hive_gcs"; + public static final String HIVE4 = "hive4"; public static final String HIVE_SPARK = "hive_spark"; public static final String HIVE_SPARK_NO_STATS_FALLBACK = "hive_spark_no_stats_fallback"; public static final String HIVE_COMPRESSION = "hive_compression"; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java index 9adf421d384a..fea4732910c0 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java @@ -22,6 +22,7 @@ import io.trino.tests.product.launcher.env.common.HadoopKerberos; import io.trino.tests.product.launcher.env.common.HadoopKerberosKms; import io.trino.tests.product.launcher.env.common.HadoopKerberosKmsWithImpersonation; +import io.trino.tests.product.launcher.env.common.Hive4WithMinio; import io.trino.tests.product.launcher.env.common.HttpProxy; import io.trino.tests.product.launcher.env.common.HttpsProxy; import io.trino.tests.product.launcher.env.common.HydraIdentityProvider; @@ -99,6 +100,7 @@ public void configure(Binder binder) binder.bind(OpenLdapReferral.class).in(SINGLETON); binder.bind(HttpProxy.class).in(SINGLETON); binder.bind(HttpsProxy.class).in(SINGLETON); + binder.bind(Hive4WithMinio.class).in(SINGLETON); MapBinder environments = newMapBinder(binder, String.class, EnvironmentProvider.class); findEnvironmentsByBasePackage(ENVIRONMENT_PACKAGE).forEach(clazz -> environments.addBinding(nameForEnvironmentClass(clazz)).to(clazz).in(SINGLETON)); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java new file mode 100644 index 000000000000..4ee02eb93028 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.common; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.List; +import java.util.Set; + +import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; +import static io.trino.tests.product.launcher.env.common.Minio.MINIO_CONTAINER_NAME; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +public class Hive4WithMinio + implements EnvironmentExtender +{ + public static final String METASTORE = "metastore"; + public static final String HIVESERVER2 = "hiveserver2"; + private static final int HIVE_SERVER_PORT = 10000; + private static final int HIVE_METASTORE_PORT = 9083; + private static final String APACHE_HIVE_IMAGE = "ghcr.io/trinodb/testing/hive4.0-hive"; + private static final File HIVE_JDBC_PROVIDER = new File("testing/trino-product-tests-launcher/target/hive-jdbc.jar"); + private static final String S3_BUCKET_NAME = "test-bucket"; + + private final PortBinder portBinder; + private final String hadoopImagesVersion; + private final DockerFiles.ResourceProvider configDir; + private final Minio minio; + + @Inject + public Hive4WithMinio( + DockerFiles dockerFiles, + PortBinder portBinder, + EnvironmentConfig config, + Minio minio) + { + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + this.hadoopImagesVersion = requireNonNull(config, "config is null").getHadoopImagesVersion(); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("common/hive4-with-minio"); + this.minio = requireNonNull(minio, "minio is null"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.addContainer(createMetastoreServer()); + builder.addContainer(createHiveserver2()); + builder.containerDependsOn(HIVESERVER2, METASTORE); + + configureMinio(builder); + configureTests(builder); + configureTempto(builder, configDir); + } + + @Override + public List getDependencies() + { + return ImmutableList.of(minio); + } + + private DockerContainer createMetastoreServer() + { + DockerContainer container = new DockerContainer(APACHE_HIVE_IMAGE + ":" + hadoopImagesVersion, METASTORE) + .withEnv("SERVICE_NAME", "metastore") + .withCopyFileToContainer( + forHostPath(configDir.getPath("hive-site.xml")), + "/opt/hive/conf/hive-site.xml") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(Wait.forListeningPort()) + .withStartupTimeout(Duration.ofMinutes(5)); + + portBinder.exposePort(container, HIVE_METASTORE_PORT); + return container; + } + + private DockerContainer createHiveserver2() + { + DockerContainer container = new DockerContainer(APACHE_HIVE_IMAGE + ":" + hadoopImagesVersion, HIVESERVER2) + .withEnv("SERVICE_NAME", "hiveserver2") + .withEnv("SERVICE_OPTS", "-Xmx1G -Dhive.metastore.uris=%s".formatted(URI.create("thrift://%s:%d".formatted(METASTORE, HIVE_METASTORE_PORT)))) + .withEnv("IS_RESUME", "true") + .withEnv("AWS_ACCESS_KEY_ID", "minio-access-key") + .withEnv("AWS_SECRET_KEY", "minio-secret-key") + .withCopyFileToContainer( + forHostPath(configDir.getPath("hive-site.xml")), + "/opt/hive/conf/hive-site.xml") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(forSelectedPorts(HIVE_SERVER_PORT)) + .withStartupTimeout(Duration.ofMinutes(5)); + + portBinder.exposePort(container, HIVE_SERVER_PORT); + return container; + } + + private void configureMinio(Environment.Builder builder) + { + FileAttribute> posixFilePermissions = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")); + Path minioBucketDirectory; + try { + minioBucketDirectory = Files.createTempDirectory("test-bucket-contents", posixFilePermissions); + minioBucketDirectory.toFile().deleteOnExit(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + builder.configureContainer(MINIO_CONTAINER_NAME, container -> + container.withCopyFileToContainer(forHostPath(minioBucketDirectory), "/data/" + S3_BUCKET_NAME)); + } + + private void configureTests(Environment.Builder builder) + { + builder.configureContainer(TESTS, dockerContainer -> + dockerContainer + .withEnv("S3_BUCKET", S3_BUCKET_NAME) + .withCopyFileToContainer(forHostPath(HIVE_JDBC_PROVIDER.getAbsolutePath()), "/docker/jdbc/hive-jdbc.jar")); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java new file mode 100644 index 000000000000..8c8e0423334a --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.Hive4WithMinio; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeHive4 + extends EnvironmentProvider +{ + private final DockerFiles.ResourceProvider configDir; + + @Inject + public EnvMultinodeHive4(StandardMultinode standardMultinode, DockerFiles dockerFiles, Hive4WithMinio hive4WithMinio) + { + super(ImmutableList.of(standardMultinode, hive4WithMinio)); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/multinode-hive4"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("trino/catalog/hive.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java new file mode 100644 index 000000000000..559e9d90bcc1 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.suite.suites; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeHive4; +import io.trino.tests.product.launcher.suite.Suite; +import io.trino.tests.product.launcher.suite.SuiteTestRun; + +import java.util.List; + +import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; +import static io.trino.tests.product.TestGroups.HIVE4; +import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; + +public class SuiteHive4 + extends Suite +{ + @Override + public List getTestRuns(EnvironmentConfig config) + { + return ImmutableList.of( + testOnEnvironment(EnvMultinodeHive4.class) + .withGroups(HIVE4, CONFIGURED_FEATURES) + .build()); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml new file mode 100644 index 000000000000..ca322685d25c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml @@ -0,0 +1,80 @@ + + + + hive.server2.enable.doAs + false + + + hive.tez.exec.inplace.progress + false + + + hive.exec.scratchdir + /opt/hive/scratch_dir + + + hive.user.install.directory + /opt/hive/install_dir + + + tez.runtime.optimize.local.fetch + true + + + hive.exec.submit.local.task.via.child + false + + + mapreduce.framework.name + local + + + tez.local.mode + true + + + hive.execution.engine + tez + + + hive.metastore.warehouse.dir + s3a://test-bucket/ + + + metastore.metastore.event.db.notification.api.auth + false + + + + + hive.users.in.admin.role + hive + + + + + fs.s3a.access.key + minio-access-key + + + fs.s3a.secret.key + minio-secret-key + + + fs.s3a.endpoint + http://minio:9080 + + + fs.s3a.path.style.access + true + + + fs.s3.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3n.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml new file mode 100644 index 000000000000..c16b1d16a079 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml @@ -0,0 +1,18 @@ +databases: + hive: + host: hiveserver2 + jdbc_driver_class: org.apache.hive.jdbc.HiveDriver + jdbc_url: jdbc:hive2://${databases.hive.host}:10000 + jdbc_user: hive + jdbc_password: na + jdbc_pooling: false + schema: default + prepare_statement: + - USE ${databases.hive.schema} + # Hive 4 gathers stats by default. For test purposes we need to disable this behavior. + - SET hive.stats.column.autogather=false + table_manager_type: hive + warehouse_directory_path: s3a://${S3_BUCKET}/ + metastore: + host: metastore + port: 9083 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties new file mode 100644 index 000000000000..9f1d8a437c84 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties @@ -0,0 +1,13 @@ +connector.name=hive +hive.metastore.uri=thrift://metastore:9083 +hive.non-managed-table-writes-enabled=true +fs.native-s3.enabled=true +fs.hadoop.enabled=false +s3.region=us-east-1 +s3.aws-access-key=minio-access-key +s3.aws-secret-key=minio-secret-key +s3.endpoint=http://minio:9080/ +s3.path-style-access=true + +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java new file mode 100644 index 000000000000..cea5ca741e16 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.hive; + +import io.trino.tempto.ProductTest; +import io.trino.tempto.assertions.QueryAssert; +import org.testng.annotations.Test; + +import java.sql.Date; +import java.sql.Timestamp; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.tests.product.TestGroups.HIVE4; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onHive; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHiveOnOrcLegacyDateCompatibility + extends ProductTest +{ + private static final String TRINO_CATALOG = "hive"; + private static final String SCHEMA = "default"; + + @Test(groups = {HIVE4, PROFILE_SPECIFIC_TESTS}) + public void testReadLegacyDateFromOrcWrittenByTrino() + { + String hiveTableName = "test_hive_orc_legacy_date_compatibility_%s".formatted(randomNameSuffix()); + String trinoTableName = format("%s.%s.%s", TRINO_CATALOG, SCHEMA, hiveTableName); + + try { + onTrino().executeQuery("CREATE TABLE %s (date_col date, t timestamp) WITH (format = 'ORC')".formatted(trinoTableName)); + onTrino().executeQuery(""" + INSERT INTO %s VALUES + (DATE '0002-01-01', TIMESTAMP '0002-01-01 00:00:00.123'), + (DATE '1500-01-01', TIMESTAMP '1500-01-01 00:00:00.123'), + (DATE '1582-10-04', TIMESTAMP '1582-10-04 00:00:00.123'), + (DATE '1582-11-04', TIMESTAMP '1582-11-04 00:00:00.123'), + (DATE '2000-02-29', TIMESTAMP '2000-02-29 00:00:00.123')""".formatted(trinoTableName)); + + QueryAssert.Row[] expectedRows = { + row(Date.valueOf("0002-01-01"), Timestamp.valueOf("0002-01-01 00:00:00.123")), + row(Date.valueOf("1500-01-01"), Timestamp.valueOf("1500-01-01 00:00:00.123")), + row(Date.valueOf("1582-10-04"), Timestamp.valueOf("1582-10-04 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + + assertThat(onTrino().executeQuery("SELECT date_col, t FROM " + trinoTableName)).containsOnly(expectedRows); + assertThat(onHive().executeQuery("SELECT date_col, t FROM " + hiveTableName)).containsOnly(expectedRows); + } + finally { + onTrino().executeQuery("DROP TABLE IF EXISTS " + trinoTableName); + } + } + + @Test(groups = {HIVE4, PROFILE_SPECIFIC_TESTS}) + public void testReadLegacyDateFromOrcWrittenByHive() + { + String hiveTableName = "test_hive_orc_legacy_date_compatibility_%s".formatted(randomNameSuffix()); + String trinoTableName = format("%s.%s.%s", TRINO_CATALOG, SCHEMA, hiveTableName); + + try { + onHive().executeQuery("CREATE TABLE %s (date_col date, t timestamp) STORED AS ORC".formatted(hiveTableName)); + onHive().executeQuery(""" + INSERT INTO %s VALUES + ('0002-01-01', '0002-01-01 00:00:00.123'), + ('1500-01-01', '1500-01-01 00:00:00.123'), + ('1582-10-04', '1582-10-04 00:00:00.123'), + ('1582-11-04', '1582-11-04 00:00:00.123'), + ('2000-02-29', '2000-02-29 00:00:00.123')""".formatted(hiveTableName)); + + QueryAssert.Row[] expectedRowsHive = { + row(Date.valueOf("0002-01-01"), Timestamp.valueOf("0002-01-01 00:00:00.123")), + row(Date.valueOf("1500-01-01"), Timestamp.valueOf("1500-01-01 00:00:00.123")), + row(Date.valueOf("1582-10-04"), Timestamp.valueOf("1582-10-04 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + assertThat(onHive().executeQuery("SELECT date_col, t FROM " + hiveTableName)).containsOnly(expectedRowsHive); + + // https://github.com/trinodb/trino/issues/26865 + QueryAssert.Row[] expectedRowsTrino = { + row(Date.valueOf("0001-12-30"), Timestamp.valueOf("0001-12-30 00:00:00.123")), + row(Date.valueOf("1500-01-10"), Timestamp.valueOf("1500-01-10 00:00:00.123")), + row(Date.valueOf("1582-10-24"), Timestamp.valueOf("1582-10-24 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + assertThat(onTrino().executeQuery("SELECT date_col, t FROM " + trinoTableName)).containsOnly(expectedRowsTrino); + } + finally { + onHive().executeQuery("DROP TABLE IF EXISTS " + trinoTableName); + } + } +}