diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json index 34a6e02150e7..5d04b2c0a8c7 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 4 + "modification": 5 } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java index 103df541e50b..4645be9bfde8 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java @@ -27,31 +27,20 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.beam.sdk.coders.KvCoder; import org.apache.beam.sdk.coders.VarIntCoder; import org.apache.beam.sdk.coders.VarLongCoder; -import org.apache.beam.sdk.io.Compression; -import org.apache.beam.sdk.io.FileSystems; -import org.apache.beam.sdk.io.fs.ResourceId; -import org.apache.beam.sdk.io.parquet.ParquetIO.ReadFiles.BeamParquetInputFile; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.SchemaCoder; @@ -75,8 +64,6 @@ import org.apache.beam.sdk.values.TupleTagList; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings; -import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables; -import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hasher; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; import org.apache.iceberg.AppendFiles; @@ -111,7 +98,6 @@ import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; -import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; @@ -251,15 +237,15 @@ public PCollectionRowTuple expand(PCollection input) { *

Asynchronous Bundle Processing: Because file I/O, catalog lookups, and metadata * inference can be highly latency-bound, this DoFn implements an asynchronous processing pattern * to maximize throughput. By default, Beam processes elements in a bundle sequentially. To avoid - * bottlenecking the pipeline, we use an internal {@link ExecutorService} to process multiple + * bottlenecking the pipeline, we use an internal {@link BoundedAsyncTasks} to process multiple * files concurrently within a single DoFn instance. * *

Lifecycle & Thread Safety: * *