httpClientProperties) {
this.proxyEndpoint =
PropertyUtil.propertyAsString(
httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
+ this.proxyUseSystemPropertyValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES);
+ this.proxyUseEnvironmentVariableValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES);
}
@VisibleForTesting
@@ -113,9 +121,26 @@ void configureApacheHttpClientBuilder(ApacheHttpClient.Builder apacheHttpClientB
if (useIdleConnectionReaperEnabled != null) {
apacheHttpClientBuilder.useIdleConnectionReaper(useIdleConnectionReaperEnabled);
}
- if (proxyEndpoint != null) {
- apacheHttpClientBuilder.proxyConfiguration(
- ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ configureProxy(apacheHttpClientBuilder);
+ }
+
+ private void configureProxy(ApacheHttpClient.Builder apacheHttpClientBuilder) {
+ if (proxyEndpoint != null
+ || proxyUseSystemPropertyValues != null
+ || proxyUseEnvironmentVariableValues != null) {
+ ProxyConfiguration.Builder proxyBuilder = ProxyConfiguration.builder();
+
+ if (proxyEndpoint != null) {
+ proxyBuilder.endpoint(URI.create(proxyEndpoint));
+ }
+ if (proxyUseSystemPropertyValues != null) {
+ proxyBuilder.useSystemPropertyValues(proxyUseSystemPropertyValues);
+ }
+ if (proxyUseEnvironmentVariableValues != null) {
+ proxyBuilder.useEnvironmentVariableValues(proxyUseEnvironmentVariableValues);
+ }
+
+ apacheHttpClientBuilder.proxyConfiguration(proxyBuilder.build());
}
}
@@ -138,6 +163,8 @@ protected String generateHttpClientCacheKey() {
keyComponents.put("tcpKeepAliveEnabled", tcpKeepAliveEnabled);
keyComponents.put("useIdleConnectionReaperEnabled", useIdleConnectionReaperEnabled);
keyComponents.put("proxyEndpoint", proxyEndpoint);
+ keyComponents.put("proxyUseSystemPropertyValues", proxyUseSystemPropertyValues);
+ keyComponents.put("proxyUseEnvironmentVariableValues", proxyUseEnvironmentVariableValues);
return keyComponents.entrySet().stream()
.map(entry -> entry.getKey() + "=" + Objects.toString(entry.getValue(), "null"))
diff --git a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
index 438ae5bb0431..870d8e23651c 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
@@ -61,6 +61,30 @@ public class HttpClientProperties implements Serializable {
*/
public static final String PROXY_ENDPOINT = "http-client.proxy-endpoint";
+ /**
+ * Used to enable reading proxy configuration from Java system properties (http.proxyHost,
+ * http.proxyPort, http.nonProxyHosts, etc.). Default is true.
+ *
+ * For more details, see
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/ProxyConfiguration.html
+ * and
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ProxyConfiguration.html
+ */
+ public static final String PROXY_USE_SYSTEM_PROPERTY_VALUES =
+ "http-client.proxy-use-system-property-values";
+
+ /**
+ * Used to enable reading proxy configuration from environment variables (HTTP_PROXY, HTTPS_PROXY,
+ * NO_PROXY, etc.). Default is true.
+ *
+ *
For more details, see
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/ProxyConfiguration.html
+ * and
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ProxyConfiguration.html
+ */
+ public static final String PROXY_USE_ENVIRONMENT_VARIABLE_VALUES =
+ "http-client.proxy-use-environment-variable-values";
+
/**
* Used to configure the connection timeout in milliseconds for {@link
* software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder}. This flag only
diff --git a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
index 98808ead4f0b..48281841be37 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
@@ -18,12 +18,15 @@
*/
package org.apache.iceberg.aws;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
+import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.rest.HTTPHeaders;
import org.apache.iceberg.rest.HTTPHeaders.HTTPHeader;
@@ -64,16 +67,23 @@ public class RESTSigV4AuthSession implements AuthSession {
private final Region signingRegion;
private final String signingName;
private final AwsCredentialsProvider credentialsProvider;
+ private final CloseableGroup closeableGroup;
@SuppressWarnings("deprecation")
public RESTSigV4AuthSession(
Aws4Signer aws4Signer, AuthSession delegateAuthSession, AwsProperties awsProperties) {
+ this.closeableGroup = new CloseableGroup();
+ this.closeableGroup.setSuppressCloseFailure(true);
this.signer = Preconditions.checkNotNull(aws4Signer, "Invalid signer: null");
this.delegate = Preconditions.checkNotNull(delegateAuthSession, "Invalid delegate: null");
+ this.closeableGroup.addCloseable(this.delegate);
Preconditions.checkNotNull(awsProperties, "Invalid AWS properties: null");
this.signingRegion = awsProperties.restSigningRegion();
this.signingName = awsProperties.restSigningName();
this.credentialsProvider = awsProperties.restCredentialsProvider();
+ if (credentialsProvider instanceof AutoCloseable closeableCredentialsProvider) {
+ this.closeableGroup.addCloseable(closeableCredentialsProvider);
+ }
}
public AuthSession delegate() {
@@ -87,7 +97,11 @@ public HTTPRequest authenticate(HTTPRequest request) {
@Override
public void close() {
- delegate.close();
+ try {
+ closeableGroup.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
}
@SuppressWarnings("deprecation")
diff --git a/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
index 4aec0bda2a13..3306163baffd 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
@@ -32,7 +32,7 @@ private S3FileIOAwsClientFactories() {}
/**
* Attempts to load an AWS client factory class for S3 file IO defined in the catalog property
* {@link S3FileIOProperties#CLIENT_FACTORY}. If the property wasn't set, fallback to {@link
- * AwsClientFactories#from(Map) to intialize an AWS client factory class}
+ * AwsClientFactories#from(Map) to initialize an AWS client factory class}
*
* @param properties catalog properties
* @return an instance of a factory class
diff --git a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
index 273baa674804..fbd845852ca9 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
@@ -35,6 +35,8 @@ class UrlConnectionHttpClientConfigurations extends BaseHttpClientConfigurations
private Long httpClientUrlConnectionConnectionTimeoutMs;
private Long httpClientUrlConnectionSocketTimeoutMs;
private String proxyEndpoint;
+ private Boolean proxyUseSystemPropertyValues;
+ private Boolean proxyUseEnvironmentVariableValues;
private UrlConnectionHttpClientConfigurations() {}
@@ -56,6 +58,12 @@ private void initialize(Map httpClientProperties) {
this.proxyEndpoint =
PropertyUtil.propertyAsString(
httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
+ this.proxyUseSystemPropertyValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES);
+ this.proxyUseEnvironmentVariableValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES);
}
@VisibleForTesting
@@ -69,9 +77,26 @@ void configureUrlConnectionHttpClientBuilder(
urlConnectionHttpClientBuilder.socketTimeout(
Duration.ofMillis(httpClientUrlConnectionSocketTimeoutMs));
}
- if (proxyEndpoint != null) {
- urlConnectionHttpClientBuilder.proxyConfiguration(
- ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ configureProxy(urlConnectionHttpClientBuilder);
+ }
+
+ private void configureProxy(UrlConnectionHttpClient.Builder urlConnectionHttpClientBuilder) {
+ if (proxyEndpoint != null
+ || proxyUseSystemPropertyValues != null
+ || proxyUseEnvironmentVariableValues != null) {
+ ProxyConfiguration.Builder proxyBuilder = ProxyConfiguration.builder();
+
+ if (proxyEndpoint != null) {
+ proxyBuilder.endpoint(URI.create(proxyEndpoint));
+ }
+ if (proxyUseSystemPropertyValues != null) {
+ proxyBuilder.useSystemPropertyValues(proxyUseSystemPropertyValues);
+ }
+ if (proxyUseEnvironmentVariableValues != null) {
+ proxyBuilder.useEnvironmentVariablesValues(proxyUseEnvironmentVariableValues);
+ }
+
+ urlConnectionHttpClientBuilder.proxyConfiguration(proxyBuilder.build());
}
}
@@ -87,6 +112,8 @@ protected String generateHttpClientCacheKey() {
keyComponents.put("connectionTimeoutMs", httpClientUrlConnectionConnectionTimeoutMs);
keyComponents.put("socketTimeoutMs", httpClientUrlConnectionSocketTimeoutMs);
keyComponents.put("proxyEndpoint", proxyEndpoint);
+ keyComponents.put("proxyUseSystemPropertyValues", proxyUseSystemPropertyValues);
+ keyComponents.put("proxyUseEnvironmentVariableValues", proxyUseEnvironmentVariableValues);
return keyComponents.entrySet().stream()
.map(entry -> entry.getKey() + "=" + Objects.toString(entry.getValue(), "null"))
diff --git a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
index 0c991af75076..7c75f99d6d69 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
@@ -53,6 +53,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.util.LocationUtil;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +113,7 @@ public class DynamoDbCatalog extends BaseMetastoreCatalog
private FileIO fileIO;
private CloseableGroup closeableGroup;
private Map catalogProperties;
+ private boolean uniqueTableLocation;
public DynamoDbCatalog() {}
@@ -123,12 +125,21 @@ public void initialize(String name, Map properties) {
properties.get(CatalogProperties.WAREHOUSE_LOCATION),
new AwsProperties(properties),
AwsClientFactories.from(properties).dynamo(),
- initializeFileIO(properties));
+ initializeFileIO(properties),
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
@VisibleForTesting
void initialize(
- String name, String path, AwsProperties properties, DynamoDbClient client, FileIO io) {
+ String name,
+ String path,
+ AwsProperties properties,
+ DynamoDbClient client,
+ FileIO io,
+ boolean uniqTableLocation) {
Preconditions.checkArgument(
!Strings.isNullOrEmpty(path),
"Cannot initialize DynamoDbCatalog because warehousePath must not be null or empty");
@@ -138,6 +149,7 @@ void initialize(
this.warehousePath = LocationUtil.stripTrailingSlash(path);
this.dynamo = client;
this.fileIO = io;
+ this.uniqueTableLocation = uniqTableLocation;
this.closeableGroup = new CloseableGroup();
closeableGroup.addCloseable(dynamo);
@@ -177,12 +189,12 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
}
String defaultLocationCol = toPropertyCol(PROPERTY_DEFAULT_LOCATION);
+ String tableLocation = LocationUtil.tableLocation(tableIdentifier, uniqueTableLocation);
if (response.item().containsKey(defaultLocationCol)) {
- return String.format(
- "%s/%s", response.item().get(defaultLocationCol).s(), tableIdentifier.name());
+ return String.format("%s/%s", response.item().get(defaultLocationCol).s(), tableLocation);
} else {
return String.format(
- "%s/%s.db/%s", warehousePath, tableIdentifier.namespace(), tableIdentifier.name());
+ "%s/%s.db/%s", warehousePath, tableIdentifier.namespace(), tableLocation);
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
index 47807a2b9f37..94e53cc1ab69 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
@@ -89,6 +89,7 @@ public class GlueCatalog extends BaseMetastoreCatalog
private Object hadoopConf;
private String catalogName;
private String warehousePath;
+ private boolean uniqueTableLocation;
private AwsProperties awsProperties;
private S3FileIOProperties s3FileIOProperties;
private LockManager lockManager;
@@ -144,7 +145,11 @@ public void initialize(String name, Map properties) {
new AwsProperties(properties),
new S3FileIOProperties(properties),
awsClientFactory.glue(),
- initializeLockManager(properties));
+ initializeLockManager(properties),
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
private LockManager initializeLockManager(Map properties) {
@@ -172,7 +177,17 @@ void initialize(
LockManager lock,
Map catalogProps) {
this.catalogProperties = catalogProps;
- initialize(name, path, properties, s3Properties, client, lock);
+ initialize(
+ name,
+ path,
+ properties,
+ s3Properties,
+ client,
+ lock,
+ PropertyUtil.propertyAsBoolean(
+ catalogProps,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
@VisibleForTesting
@@ -182,13 +197,15 @@ void initialize(
AwsProperties properties,
S3FileIOProperties s3Properties,
GlueClient client,
- LockManager lock) {
+ LockManager lock,
+ boolean uniqTableLocation) {
this.catalogName = name;
this.awsProperties = properties;
this.s3FileIOProperties = s3Properties;
this.warehousePath = Strings.isNullOrEmpty(path) ? null : LocationUtil.stripTrailingSlash(path);
this.glue = client;
this.lockManager = lock;
+ this.uniqueTableLocation = uniqTableLocation;
this.closeableGroup = new CloseableGroup();
this.fileIOTracker = new FileIOTracker();
@@ -278,9 +295,10 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
tableIdentifier, awsProperties.glueCatalogSkipNameValidation()))
.build());
String dbLocationUri = response.database().locationUri();
+ String tableLocation = LocationUtil.tableLocation(tableIdentifier, uniqueTableLocation);
if (dbLocationUri != null) {
dbLocationUri = LocationUtil.stripTrailingSlash(dbLocationUri);
- return String.format("%s/%s", dbLocationUri, tableIdentifier.name());
+ return String.format("%s/%s", dbLocationUri, tableLocation);
}
ValidationException.check(
@@ -292,7 +310,7 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
warehousePath,
IcebergToGlueConverter.getDatabaseName(
tableIdentifier, awsProperties.glueCatalogSkipNameValidation()),
- tableIdentifier.name());
+ tableLocation);
}
@Override
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
index ad5181fd2798..922010d61d27 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
@@ -295,6 +295,18 @@ public class S3FileIOProperties implements Serializable {
public static final boolean REMOTE_SIGNING_ENABLED_DEFAULT = false;
+ /**
+ * Enables or disables chunked encoding for S3 requests.
+ *
+ * This feature is enabled by default to match the AWS SDK default behavior.
+ *
+ *
For more details see:
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#chunkedEncodingEnabled()
+ */
+ public static final String CHUNKED_ENCODING_ENABLED = "s3.chunked-encoding-enabled";
+
+ public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = true;
+
/** Configure the batch size used when deleting multiple files from a given S3 bucket */
public static final String DELETE_BATCH_SIZE = "s3.delete.batch-size";
@@ -509,6 +521,7 @@ public class S3FileIOProperties implements Serializable {
private String stagingDirectory;
private ObjectCannedACL acl;
private boolean isChecksumEnabled;
+ private boolean isChunkedEncodingEnabled;
private final Set writeTags;
private boolean isWriteTableTagEnabled;
private boolean isWriteNamespaceTagEnabled;
@@ -551,6 +564,7 @@ public S3FileIOProperties() {
this.deleteBatchSize = DELETE_BATCH_SIZE_DEFAULT;
this.stagingDirectory = System.getProperty("java.io.tmpdir");
this.isChecksumEnabled = CHECKSUM_ENABLED_DEFAULT;
+ this.isChunkedEncodingEnabled = CHUNKED_ENCODING_ENABLED_DEFAULT;
this.writeTags = Sets.newHashSet();
this.isWriteTableTagEnabled = WRITE_TABLE_TAG_ENABLED_DEFAULT;
this.isWriteNamespaceTagEnabled = WRITE_NAMESPACE_TAG_ENABLED_DEFAULT;
@@ -641,6 +655,9 @@ public S3FileIOProperties(Map properties) {
"Cannot support S3 CannedACL " + aclType);
this.isChecksumEnabled =
PropertyUtil.propertyAsBoolean(properties, CHECKSUM_ENABLED, CHECKSUM_ENABLED_DEFAULT);
+ this.isChunkedEncodingEnabled =
+ PropertyUtil.propertyAsBoolean(
+ properties, CHUNKED_ENCODING_ENABLED, CHUNKED_ENCODING_ENABLED_DEFAULT);
this.deleteBatchSize =
PropertyUtil.propertyAsInt(properties, DELETE_BATCH_SIZE, DELETE_BATCH_SIZE_DEFAULT);
Preconditions.checkArgument(
@@ -808,6 +825,10 @@ public boolean isChecksumEnabled() {
return this.isChecksumEnabled;
}
+ public boolean isChunkedEncodingEnabled() {
+ return this.isChunkedEncodingEnabled;
+ }
+
public boolean isRemoteSigningEnabled() {
return this.isRemoteSigningEnabled;
}
@@ -994,6 +1015,7 @@ public void applyServiceConfigurations(T builder) {
.pathStyleAccessEnabled(isPathStyleAccess)
.useArnRegionEnabled(isUseArnRegionEnabled)
.accelerateModeEnabled(isAccelerationEnabled)
+ .chunkedEncodingEnabled(isChunkedEncodingEnabled)
.build());
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
index 89145b2465e5..7f1d6c3cc848 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
@@ -40,6 +40,10 @@
import org.apache.iceberg.rest.responses.ErrorResponse;
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@code RESTObjectMapper} instead.
+ */
+@Deprecated
public class S3ObjectMapper {
private static final JsonFactory FACTORY = new JsonFactory();
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
index 879ce8599352..995f6e7e4860 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
@@ -18,31 +18,13 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-import org.apache.iceberg.rest.RESTRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
import org.immutables.value.Value;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequest} instead.
+ */
+@Deprecated
@Value.Immutable
-public interface S3SignRequest extends RESTRequest {
- String region();
-
- String method();
-
- URI uri();
-
- Map> headers();
-
- Map properties();
-
- @Value.Default
- @Nullable
- default String body() {
- return null;
- }
-
- @Override
- default void validate() {}
-}
+@SuppressWarnings("immutables:subtype")
+public interface S3SignRequest extends RemoteSignRequest {}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
index 3b5eb83612e2..5d2a7d684460 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
@@ -21,108 +21,47 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.util.JsonUtil;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequestParser;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequestParser} instead.
+ */
+@Deprecated
public class S3SignRequestParser {
- private static final String REGION = "region";
- private static final String METHOD = "method";
- private static final String URI = "uri";
- private static final String HEADERS = "headers";
- private static final String PROPERTIES = "properties";
- private static final String BODY = "body";
-
private S3SignRequestParser() {}
public static String toJson(S3SignRequest request) {
- return toJson(request, false);
+ return RemoteSignRequestParser.toJson(request, false);
}
public static String toJson(S3SignRequest request, boolean pretty) {
- return JsonUtil.generate(gen -> toJson(request, gen), pretty);
+ return RemoteSignRequestParser.toJson(request, pretty);
}
public static void toJson(S3SignRequest request, JsonGenerator gen) throws IOException {
- Preconditions.checkArgument(null != request, "Invalid s3 sign request: null");
-
- gen.writeStartObject();
-
- gen.writeStringField(REGION, request.region());
- gen.writeStringField(METHOD, request.method());
- gen.writeStringField(URI, request.uri().toString());
- headersToJson(HEADERS, request.headers(), gen);
-
- if (!request.properties().isEmpty()) {
- JsonUtil.writeStringMap(PROPERTIES, request.properties(), gen);
- }
-
- if (request.body() != null && !request.body().isEmpty()) {
- gen.writeStringField(BODY, request.body());
- }
-
- gen.writeEndObject();
+ RemoteSignRequestParser.toJson(request, gen);
}
public static S3SignRequest fromJson(String json) {
- return JsonUtil.parse(json, S3SignRequestParser::fromJson);
+ RemoteSignRequest request = RemoteSignRequestParser.fromJson(json);
+ return ImmutableS3SignRequest.builder().from(request).build();
}
public static S3SignRequest fromJson(JsonNode json) {
- Preconditions.checkArgument(null != json, "Cannot parse s3 sign request from null object");
- Preconditions.checkArgument(
- json.isObject(), "Cannot parse s3 sign request from non-object: %s", json);
-
- String region = JsonUtil.getString(REGION, json);
- String method = JsonUtil.getString(METHOD, json);
- java.net.URI uri = java.net.URI.create(JsonUtil.getString(URI, json));
- Map> headers = headersFromJson(HEADERS, json);
-
- ImmutableS3SignRequest.Builder builder =
- ImmutableS3SignRequest.builder().region(region).method(method).uri(uri).headers(headers);
-
- if (json.has(PROPERTIES)) {
- builder.properties(JsonUtil.getStringMap(PROPERTIES, json));
- }
-
- if (json.has(BODY)) {
- builder.body(JsonUtil.getString(BODY, json));
- }
-
- return builder.build();
+ RemoteSignRequest request = RemoteSignRequestParser.fromJson(json);
+ return ImmutableS3SignRequest.builder().from(request).build();
}
static void headersToJson(String property, Map> headers, JsonGenerator gen)
throws IOException {
- gen.writeObjectFieldStart(property);
- for (Entry> entry : headers.entrySet()) {
- gen.writeFieldName(entry.getKey());
-
- gen.writeStartArray();
- for (String val : entry.getValue()) {
- gen.writeString(val);
- }
- gen.writeEndArray();
- }
- gen.writeEndObject();
+ RemoteSignRequestParser.headersToJson(property, headers, gen);
}
static Map> headersFromJson(String property, JsonNode json) {
- Map> headers = Maps.newHashMap();
- JsonNode headersNode = JsonUtil.get(property, json);
- headersNode
- .properties()
- .forEach(
- entry -> {
- String key = entry.getKey();
- List values = Arrays.asList(JsonUtil.getStringArray(entry.getValue()));
- headers.put(key, values);
- });
- return headers;
+ return RemoteSignRequestParser.headersFromJson(property, json);
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
index 40c2059488f8..6fbaa90fe7af 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
@@ -18,18 +18,13 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import org.apache.iceberg.rest.RESTResponse;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
import org.immutables.value.Value;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponse} instead.
+ */
+@Deprecated
@Value.Immutable
-public interface S3SignResponse extends RESTResponse {
- URI uri();
-
- Map> headers();
-
- @Override
- default void validate() {}
-}
+@SuppressWarnings("immutables:subtype")
+public interface S3SignResponse extends RemoteSignResponse {}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
index 69d6de8f04ac..be63a51b38fb 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
@@ -21,49 +21,37 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.util.JsonUtil;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
+import org.apache.iceberg.rest.responses.RemoteSignResponseParser;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponseParser}
+ * instead.
+ */
+@Deprecated
public class S3SignResponseParser {
- private static final String URI = "uri";
- private static final String HEADERS = "headers";
-
private S3SignResponseParser() {}
- public static String toJson(S3SignResponse request) {
- return toJson(request, false);
+ public static String toJson(S3SignResponse response) {
+ return RemoteSignResponseParser.toJson(response, false);
}
- public static String toJson(S3SignResponse request, boolean pretty) {
- return JsonUtil.generate(gen -> toJson(request, gen), pretty);
+ public static String toJson(S3SignResponse response, boolean pretty) {
+ return RemoteSignResponseParser.toJson(response, pretty);
}
public static void toJson(S3SignResponse response, JsonGenerator gen) throws IOException {
- Preconditions.checkArgument(null != response, "Invalid s3 sign response: null");
-
- gen.writeStartObject();
-
- gen.writeStringField(URI, response.uri().toString());
- S3SignRequestParser.headersToJson(HEADERS, response.headers(), gen);
-
- gen.writeEndObject();
+ RemoteSignResponseParser.toJson(response, gen);
}
public static S3SignResponse fromJson(String json) {
- return JsonUtil.parse(json, S3SignResponseParser::fromJson);
+ RemoteSignResponse result = RemoteSignResponseParser.fromJson(json);
+ return ImmutableS3SignResponse.builder().from(result).build();
}
public static S3SignResponse fromJson(JsonNode json) {
- Preconditions.checkArgument(null != json, "Cannot parse s3 sign response from null object");
- Preconditions.checkArgument(
- json.isObject(), "Cannot parse s3 sign response from non-object: %s", json);
-
- java.net.URI uri = java.net.URI.create(JsonUtil.getString(URI, json));
- Map> headers = S3SignRequestParser.headersFromJson(HEADERS, json);
-
- return ImmutableS3SignResponse.builder().uri(uri).headers(headers).build();
+ RemoteSignResponse result = RemoteSignResponseParser.fromJson(json);
+ return ImmutableS3SignResponse.builder().from(result).build();
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
index 84b67bbdafc2..7a463abd3d2d 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
@@ -37,6 +37,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
+import org.apache.iceberg.rest.RESTCatalogProperties;
import org.apache.iceberg.rest.RESTClient;
import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.rest.ResourcePaths;
@@ -45,6 +46,9 @@
import org.apache.iceberg.rest.auth.AuthSession;
import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.apache.iceberg.rest.auth.OAuth2Util;
+import org.apache.iceberg.rest.requests.ImmutableRemoteSignRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
import org.apache.iceberg.util.PropertyUtil;
import org.immutables.value.Value;
import org.slf4j.Logger;
@@ -64,13 +68,30 @@ public abstract class S3V4RestSignerClient
extends AbstractAws4Signer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(S3V4RestSignerClient.class);
- public static final String S3_SIGNER_URI = "s3.signer.uri";
- public static final String S3_SIGNER_ENDPOINT = "s3.signer.endpoint";
- static final String S3_SIGNER_DEFAULT_ENDPOINT = "v1/aws/s3/sign";
- static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
- static final String CACHE_CONTROL = "Cache-Control";
- static final String CACHE_CONTROL_PRIVATE = "private";
- static final String CACHE_CONTROL_NO_CACHE = "no-cache";
+
+ public static final String S3_PROVIDER = "s3";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link
+ * RESTCatalogProperties#SIGNER_URI} instead.
+ */
+ @Deprecated public static final String S3_SIGNER_URI = "s3.signer.uri";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link
+ * RESTCatalogProperties#SIGNER_URI} instead.
+ */
+ @Deprecated public static final String S3_SIGNER_ENDPOINT = "s3.signer.endpoint";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; there is no replacement.
+ */
+ @Deprecated static final String S3_SIGNER_DEFAULT_ENDPOINT = "v1/aws/s3/sign";
+
+ @VisibleForTesting static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
+
+ private static final String CACHE_CONTROL = "Cache-Control";
+ private static final String CACHE_CONTROL_PRIVATE = "private";
private static final Cache SIGNED_COMPONENT_CACHE =
Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).maximumSize(100).build();
@@ -94,13 +115,28 @@ public Supplier> requestPropertiesSupplier() {
@Value.Lazy
public String baseSignerUri() {
- return properties().getOrDefault(S3_SIGNER_URI, properties().get(CatalogProperties.URI));
+ // TODO remove in 1.12.0
+ if (properties().containsKey(S3_SIGNER_URI)) {
+ return properties().get(S3_SIGNER_URI);
+ }
+
+ return properties()
+ .getOrDefault(RESTCatalogProperties.SIGNER_URI, properties().get(CatalogProperties.URI));
}
@Value.Lazy
public String endpoint() {
- return RESTUtil.resolveEndpoint(
- baseSignerUri(), properties().getOrDefault(S3_SIGNER_ENDPOINT, S3_SIGNER_DEFAULT_ENDPOINT));
+ // TODO remove in 1.12.0
+ String endpointPath;
+ if (properties().containsKey(S3_SIGNER_ENDPOINT)) {
+ endpointPath = properties().get(S3_SIGNER_ENDPOINT);
+ } else {
+ endpointPath =
+ properties()
+ .getOrDefault(RESTCatalogProperties.SIGNER_ENDPOINT, S3_SIGNER_DEFAULT_ENDPOINT);
+ }
+
+ return RESTUtil.resolveEndpoint(baseSignerUri(), endpointPath);
}
/** A credential to exchange for a token in the OAuth2 client credentials flow. */
@@ -160,7 +196,6 @@ private RESTClient httpClient() {
httpClient =
HTTPClient.builder(properties())
.withHeaders(RESTUtil.configHeaders(properties()))
- .withObjectMapper(S3ObjectMapper.mapper())
.build();
}
}
@@ -197,8 +232,36 @@ private boolean credentialProvided() {
@Value.Check
protected void check() {
Preconditions.checkArgument(
- properties().containsKey(S3_SIGNER_URI) || properties().containsKey(CatalogProperties.URI),
+ properties().containsKey(S3_SIGNER_URI)
+ || properties().containsKey(RESTCatalogProperties.SIGNER_URI)
+ || properties().containsKey(CatalogProperties.URI),
"S3 signer service URI is required");
+
+ if (properties().containsKey(S3_SIGNER_URI)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_URI)) {
+ LOG.warn(
+ "S3 signer URI is configured via deprecated property {}, this won't be supported in future releases. "
+ + "Please use {} instead.",
+ S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI);
+ }
+
+ if (properties().containsKey(S3_SIGNER_ENDPOINT)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) {
+ LOG.warn(
+ "Signer endpoint is configured via deprecated property {}, this won't be supported in future releases. "
+ + "Please use {} instead.",
+ S3_SIGNER_ENDPOINT,
+ RESTCatalogProperties.SIGNER_ENDPOINT);
+ }
+
+ // TODO change to required in 1.12.0
+ if (!properties().containsKey(S3_SIGNER_ENDPOINT)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) {
+ LOG.warn(
+ "Signer endpoint is not set, this won't be supported in future releases. Using deprecated default: {}",
+ S3_SIGNER_DEFAULT_ENDPOINT);
+ }
}
@Override
@@ -241,14 +304,15 @@ public SdkHttpFullRequest sign(
AwsS3V4SignerParams signerParams =
extractSignerParams(AwsS3V4SignerParams.builder(), executionAttributes).build();
- S3SignRequest remoteSigningRequest =
- ImmutableS3SignRequest.builder()
+ RemoteSignRequest remoteSigningRequest =
+ ImmutableRemoteSignRequest.builder()
.method(request.method().name())
.region(signerParams.signingRegion().id())
.uri(request.getUri())
.headers(request.headers())
.properties(requestPropertiesSupplier().get())
.body(bodyAsString(request))
+ .provider(S3_PROVIDER)
.build();
Key cacheKey = Key.from(remoteSigningRequest);
@@ -260,21 +324,21 @@ public SdkHttpFullRequest sign(
} else {
Map responseHeaders = Maps.newHashMap();
Consumer> responseHeadersConsumer = responseHeaders::putAll;
- S3SignResponse s3SignResponse =
+ RemoteSignResponse remoteSignResponse =
httpClient()
.withAuthSession(authSession())
.post(
endpoint(),
remoteSigningRequest,
- S3SignResponse.class,
+ RemoteSignResponse.class,
Map.of(),
ErrorHandlers.defaultErrorHandler(),
responseHeadersConsumer);
signedComponent =
ImmutableSignedComponent.builder()
- .headers(s3SignResponse.headers())
- .signedURI(s3SignResponse.uri())
+ .headers(remoteSignResponse.headers())
+ .signedURI(remoteSignResponse.uri())
.build();
if (canBeCached(responseHeaders)) {
@@ -351,7 +415,7 @@ interface Key {
String uri();
- static Key from(S3SignRequest request) {
+ static Key from(RemoteSignRequest request) {
return ImmutableKey.builder()
.method(request.method())
.region(request.region())
diff --git a/aws/src/main/resources/s3-signer-open-api.yaml b/aws/src/main/resources/s3-signer-open-api.yaml
index 3d719c515b2a..0b98fcc59eff 100644
--- a/aws/src/main/resources/s3-signer-open-api.yaml
+++ b/aws/src/main/resources/s3-signer-open-api.yaml
@@ -17,19 +17,22 @@
# under the License.
#
+# ⚠️ WARNING: this API is deprecated. Use the new remote signing endpoint instead,
+# see open-api/rest-catalog-open-api.yaml.
+
---
openapi: 3.0.3
info:
- title: Apache Iceberg S3 Signer API
+ title: "[DEPRECATED] Apache Iceberg S3 Signer API"
license:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.html
version: 0.0.1
description:
- Defines the specification for the S3 Signer API.
+ "[DEPRECATED] Defines the specification for the S3 Signer API."
servers:
- url: "{scheme}://{host}/{basePath}"
- description: Server URL when the port can be inferred from the scheme
+ description: "[DEPRECATED] Server URL when the port can be inferred from the scheme"
variables:
scheme:
description: The scheme of the URI, either http or https.
@@ -41,7 +44,7 @@ servers:
description: Optional prefix to be prepended to all routes
default: ""
- url: "{scheme}://{host}:{port}/{basePath}"
- description: Generic base server URL, with all parts configurable
+ description: "[DEPRECATED] Generic base server URL, with all parts configurable"
variables:
scheme:
description: The scheme of the URI, either http or https.
@@ -61,9 +64,10 @@ paths:
/v1/aws/s3/sign:
post:
+ deprecated: true
tags:
- S3 Signer API
- summary: Remotely signs S3 requests
+ summary: "[DEPRECATED] Remotely signs S3 requests"
operationId: signS3Request
requestBody:
description: The request containing the headers to be signed
@@ -95,6 +99,7 @@ components:
schemas:
S3Headers:
+ deprecated: true
type: object
additionalProperties:
type: array
@@ -102,6 +107,7 @@ components:
type: string
S3SignRequest:
+ deprecated: true
required:
- region
- uri
@@ -133,7 +139,8 @@ components:
responses:
S3SignResponse:
- description: The response containing signed & unsigned headers. The server will also send
+ description: >
+ [DEPRECATED] The response containing signed & unsigned headers. The server will also send
a Cache-Control header, indicating whether the response can be cached (Cache-Control = ["private"])
or not (Cache-Control = ["no-cache"]).
content:
diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
index 0f96ac0f6c82..da73a5c1b5a5 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestHttpClientConfigurations.java
@@ -22,6 +22,8 @@
import java.util.Map;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.Mockito;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
@@ -137,4 +139,39 @@ public void testApacheDefaultConfigurations() {
Mockito.verify(spyApacheHttpClientBuilder, Mockito.never())
.proxyConfiguration(Mockito.any(ProxyConfiguration.class));
}
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES,
+ HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES
+ })
+ public void testApacheProxyFlagTriggersProxyConfig(String propertyKey) {
+ Map properties = Maps.newHashMap();
+ properties.put(propertyKey, "false");
+ ApacheHttpClient.Builder spy = Mockito.spy(ApacheHttpClient.builder());
+
+ ApacheHttpClientConfigurations.create(properties).configureApacheHttpClientBuilder(spy);
+
+ Mockito.verify(spy).proxyConfiguration(Mockito.any(ProxyConfiguration.class));
+ }
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES,
+ HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES
+ })
+ public void testUrlConnectionProxyFlagTriggersProxyConfig(String propertyKey) {
+ Map properties = Maps.newHashMap();
+ properties.put(propertyKey, "false");
+ UrlConnectionHttpClient.Builder spy = Mockito.spy(UrlConnectionHttpClient.builder());
+
+ UrlConnectionHttpClientConfigurations.create(properties)
+ .configureUrlConnectionHttpClientBuilder(spy);
+
+ Mockito.verify(spy)
+ .proxyConfiguration(
+ Mockito.any(software.amazon.awssdk.http.urlconnection.ProxyConfiguration.class));
+ }
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestRESTSigV4AuthSession.java b/aws/src/test/java/org/apache/iceberg/aws/TestRESTSigV4AuthSession.java
index 1b2aaf2e1c01..9e996ca60089 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestRESTSigV4AuthSession.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestRESTSigV4AuthSession.java
@@ -35,7 +35,10 @@
import org.apache.iceberg.rest.requests.CreateNamespaceRequest;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.signer.Aws4Signer;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.utils.SdkAutoCloseable;
class TestRESTSigV4AuthSession {
@@ -306,4 +309,43 @@ void close() {
session.close();
Mockito.verify(delegate).close();
}
+
+ @Test
+ void closeWithCloseableCredentialsProvider() {
+ AuthSession delegate = Mockito.mock(AuthSession.class);
+ CloseableAwsCredentialsProvider credentialsProvider =
+ Mockito.mock(CloseableAwsCredentialsProvider.class);
+ closeWithCloseableCredentialsProvider(delegate, credentialsProvider);
+ }
+
+ @Test
+ void closeSuppressesFailure() {
+ AuthSession delegate = Mockito.mock(AuthSession.class);
+ Mockito.doThrow(new RuntimeException("delegate close failed")).when(delegate).close();
+ CloseableAwsCredentialsProvider credentialsProvider =
+ Mockito.mock(CloseableAwsCredentialsProvider.class);
+ Mockito.doThrow(new RuntimeException("credentials provider close failed"))
+ .when(credentialsProvider)
+ .close();
+ closeWithCloseableCredentialsProvider(delegate, credentialsProvider);
+ }
+
+ private void closeWithCloseableCredentialsProvider(
+ AuthSession delegate, CloseableAwsCredentialsProvider credentialsProvider) {
+ AwsProperties properties = Mockito.mock(AwsProperties.class);
+ when(properties.restSigningRegion()).thenReturn(Region.US_WEST_2);
+ when(properties.restSigningName()).thenReturn("execute-api");
+ when(properties.restCredentialsProvider()).thenReturn(credentialsProvider);
+
+ RESTSigV4AuthSession session = new RESTSigV4AuthSession(signer, delegate, properties);
+ session.close();
+
+ Mockito.verify(delegate).close();
+ Mockito.verify(credentialsProvider).close();
+ }
+
+ interface CloseableAwsCredentialsProvider extends AwsCredentialsProvider, SdkAutoCloseable {
+ @Override
+ void close();
+ }
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/dynamodb/TestDynamoDbCatalog.java b/aws/src/test/java/org/apache/iceberg/aws/dynamodb/TestDynamoDbCatalog.java
index b602cea303d8..e172831a2428 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/dynamodb/TestDynamoDbCatalog.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/dynamodb/TestDynamoDbCatalog.java
@@ -49,14 +49,25 @@ public class TestDynamoDbCatalog {
public void before() {
dynamo = Mockito.mock(DynamoDbClient.class);
dynamoCatalog = new DynamoDbCatalog();
- dynamoCatalog.initialize(CATALOG_NAME, WAREHOUSE_PATH, new AwsProperties(), dynamo, null);
+ dynamoCatalog.initialize(
+ CATALOG_NAME,
+ WAREHOUSE_PATH,
+ new AwsProperties(),
+ dynamo,
+ null,
+ false /* uniqTableLocation */);
}
@Test
public void testConstructorWarehousePathWithEndSlash() {
DynamoDbCatalog catalogWithSlash = new DynamoDbCatalog();
catalogWithSlash.initialize(
- CATALOG_NAME, WAREHOUSE_PATH + "/", new AwsProperties(), dynamo, null);
+ CATALOG_NAME,
+ WAREHOUSE_PATH + "/",
+ new AwsProperties(),
+ dynamo,
+ null,
+ false /* uniqTableLocation */);
Mockito.doReturn(GetItemResponse.builder().item(Maps.newHashMap()).build())
.when(dynamo)
.getItem(any(GetItemRequest.class));
@@ -103,4 +114,49 @@ public void testDefaultWarehouseLocationNoNamespace() {
.isInstanceOf(NoSuchNamespaceException.class)
.hasMessageContaining("Cannot find default warehouse location:");
}
+
+ @Test
+ public void testDefaultWarehouseLocationUniqueWithoutDbUri() throws Exception {
+ try (DynamoDbCatalog catalog = new DynamoDbCatalog()) {
+ catalog.initialize(
+ CATALOG_NAME,
+ WAREHOUSE_PATH,
+ new AwsProperties(),
+ dynamo,
+ null,
+ true /* uniqTableLocation */);
+ Mockito.doReturn(GetItemResponse.builder().item(Maps.newHashMap()).build())
+ .when(dynamo)
+ .getItem(any(GetItemRequest.class));
+
+ String defaultWarehouseLocation = catalog.defaultWarehouseLocation(TABLE_IDENTIFIER);
+ assertThat(defaultWarehouseLocation).matches(WAREHOUSE_PATH + "/db.db/table-[a-z0-9]{32}");
+ }
+ }
+
+ @Test
+ public void testDefaultWarehouseLocationUniqueWithDbUri() throws Exception {
+ try (DynamoDbCatalog catalog = new DynamoDbCatalog()) {
+ catalog.initialize(
+ CATALOG_NAME,
+ WAREHOUSE_PATH,
+ new AwsProperties(),
+ dynamo,
+ null,
+ true /* uniqTableLocation */);
+ String dbUri = "s3://bucket2/db";
+ Mockito.doReturn(
+ GetItemResponse.builder()
+ .item(
+ ImmutableMap.of(
+ toPropertyCol(DynamoDbCatalog.defaultLocationProperty()),
+ AttributeValue.builder().s(dbUri).build()))
+ .build())
+ .when(dynamo)
+ .getItem(any(GetItemRequest.class));
+
+ String defaultWarehouseLocation = catalog.defaultWarehouseLocation(TABLE_IDENTIFIER);
+ assertThat(defaultWarehouseLocation).matches("s3://bucket2/db/table-[a-z0-9]{32}");
+ }
+ }
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java
index 2042948eb3c9..82f7e84d563b 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestGlueCatalog.java
@@ -194,6 +194,28 @@ public void testDefaultWarehouseLocationCustomCatalogId() {
Mockito.argThat((GetDatabaseRequest req) -> req.catalogId().equals(catalogId)));
}
+ @Test
+ public void testDefaultWarehouseLocationUnique() {
+ GlueCatalog catalog = new GlueCatalog();
+ catalog.initialize(
+ CATALOG_NAME,
+ WAREHOUSE_PATH,
+ new AwsProperties(),
+ new S3FileIOProperties(),
+ glue,
+ LockManagers.defaultLockManager(),
+ true /* uniqTableLocation */);
+
+ Mockito.doReturn(
+ GetDatabaseResponse.builder()
+ .database(Database.builder().name("db").locationUri("s3://bucket2/db").build())
+ .build())
+ .when(glue)
+ .getDatabase(Mockito.any(GetDatabaseRequest.class));
+ String location = catalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"));
+ assertThat(location).matches("s3://bucket2/db/table-[a-z0-9]{32}");
+ }
+
@Test
public void testListTables() {
Mockito.doReturn(
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOCredentialRefresh.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOCredentialRefresh.java
index 0a8b0e084873..170857ca84b4 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOCredentialRefresh.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOCredentialRefresh.java
@@ -29,6 +29,7 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.aws.AwsProperties;
import org.apache.iceberg.io.StorageCredential;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -265,4 +266,83 @@ public void credentialRefreshWithinFiveMinuteWindow() {
});
}
}
+
+ @Test
+ public void refreshedCredentialsAreKryoSerializable() throws Exception {
+ // Verify that an S3FileIO whose credentials have been refreshed at runtime can still be
+ // round-tripped through Kryo. The internal storageCredentials list must be backed by a
+ // collection that Kryo can serialize and deserialize.
+ String nearExpiryMs = Long.toString(Instant.now().plus(3, ChronoUnit.MINUTES).toEpochMilli());
+
+ StorageCredential initialCredential =
+ StorageCredential.create(
+ "s3://bucket/path",
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "initialAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "initialSecretKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "initialToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ nearExpiryMs));
+
+ String refreshedExpiryMs =
+ Long.toString(Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli());
+ LoadCredentialsResponse refreshResponse =
+ ImmutableLoadCredentialsResponse.builder()
+ .addCredentials(
+ ImmutableCredential.builder()
+ .prefix("s3://bucket/path")
+ .config(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_KEY_ID,
+ "refreshedAccessKey",
+ S3FileIOProperties.SECRET_ACCESS_KEY,
+ "refreshedSecretKey",
+ S3FileIOProperties.SESSION_TOKEN,
+ "refreshedToken",
+ S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS,
+ refreshedExpiryMs))
+ .build())
+ .build();
+
+ HttpRequest mockRequest = request("/v1/credentials").withMethod(HttpMethod.GET.name());
+ mockServer
+ .when(mockRequest)
+ .respond(
+ response(LoadCredentialsResponseParser.toJson(refreshResponse)).withStatusCode(200));
+
+ Map properties =
+ ImmutableMap.of(
+ AwsProperties.CLIENT_FACTORY,
+ StaticClientFactory.class.getName(),
+ VendedCredentialsProvider.URI,
+ CREDENTIALS_URI,
+ CatalogProperties.URI,
+ CATALOG_URI,
+ "init-creation-stacktrace",
+ "false");
+
+ StaticClientFactory.client = null;
+ try (S3FileIO fileIO = new S3FileIO()) {
+ fileIO.initialize(properties);
+ fileIO.setCredentials(List.of(initialCredential));
+
+ fileIO.client();
+
+ // Wait for the refresh to update the in-memory credentials
+ Awaitility.await()
+ .atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertThat(fileIO.credentials().get(0).config())
+ .containsEntry(S3FileIOProperties.ACCESS_KEY_ID, "refreshedAccessKey"));
+
+ // Round-trip through Kryo and verify the credentials still match
+ try (S3FileIO deserialized = TestHelpers.KryoHelpers.roundTripSerialize(fileIO)) {
+ assertThat(deserialized.credentials()).isEqualTo(fileIO.credentials());
+ }
+ }
+ }
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
index 1666de1f1d08..953f73d45d4a 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
@@ -566,4 +566,25 @@ public void testApplyRetryConfiguration() {
RetryPolicy retryPolicy = builder.overrideConfiguration().retryPolicy().get();
assertThat(retryPolicy.numRetries()).as("retries was not set").isEqualTo(999);
}
+
+ @Test
+ public void testChunkedEncodingEnabledDefaultValue() {
+ Map properties = Maps.newHashMap();
+ S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties);
+
+ assertThat(s3FileIOProperties.isChunkedEncodingEnabled())
+ .as("chunked encoding should be enabled by default")
+ .isTrue();
+ }
+
+ @Test
+ public void testChunkedEncodingDisabled() {
+ Map properties = Maps.newHashMap();
+ properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "false");
+ S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties);
+
+ assertThat(s3FileIOProperties.isChunkedEncodingEnabled())
+ .as("chunked encoding should be disabled when explicitly set to false")
+ .isFalse();
+ }
}
diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java
index 0bcc77e29fae..aadbf036b567 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java
@@ -18,13 +18,14 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import static org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient.S3_SIGNER_URI;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.InstanceOfAssertFactories.type;
import static org.mockito.Mockito.when;
import java.util.Map;
import java.util.stream.Stream;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.rest.RESTCatalogProperties;
import org.apache.iceberg.rest.RESTClient;
import org.apache.iceberg.rest.auth.AuthProperties;
import org.apache.iceberg.rest.auth.AuthSession;
@@ -119,12 +120,21 @@ void authSessionOAuth2(Map properties, String expectedScope, Str
public static Stream validOAuth2Properties() {
return Stream.of(
// No OAuth2 data
- Arguments.of(Map.of(S3_SIGNER_URI, "https://signer.com"), "sign", null),
+ Arguments.of(
+ Map.of(
+ RESTCatalogProperties.SIGNER_URI,
+ "https://signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/sign/s3"),
+ "sign",
+ null),
// Token only
Arguments.of(
Map.of(
- S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI,
"https://signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/sign/s3",
AuthProperties.AUTH_TYPE,
AuthProperties.AUTH_TYPE_OAUTH2,
OAuth2Properties.TOKEN,
@@ -134,8 +144,10 @@ public static Stream validOAuth2Properties() {
// Credential only: expect a token to be fetched
Arguments.of(
Map.of(
- S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI,
"https://signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/sign/s3",
AuthProperties.AUTH_TYPE,
AuthProperties.AUTH_TYPE_OAUTH2,
OAuth2Properties.CREDENTIAL,
@@ -145,8 +157,10 @@ public static Stream validOAuth2Properties() {
// Token and credential: should use token as is, not fetch a new one
Arguments.of(
Map.of(
- S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI,
"https://signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/sign/s3",
AuthProperties.AUTH_TYPE,
AuthProperties.AUTH_TYPE_OAUTH2,
OAuth2Properties.TOKEN,
@@ -158,8 +172,10 @@ public static Stream validOAuth2Properties() {
// Custom scope
Arguments.of(
Map.of(
- S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI,
"https://signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/sign/s3",
AuthProperties.AUTH_TYPE,
AuthProperties.AUTH_TYPE_OAUTH2,
OAuth2Properties.CREDENTIAL,
@@ -169,4 +185,63 @@ public static Stream validOAuth2Properties() {
"custom",
"token"));
}
+
+ @ParameterizedTest
+ @MethodSource("legacySignerProperties")
+ void legacySignerProperties(
+ Map properties, String expectedBaseSignerUri, String expectedEndpoint)
+ throws Exception {
+ try (S3V4RestSignerClient client =
+ ImmutableS3V4RestSignerClient.builder().properties(properties).build()) {
+ assertThat(client.baseSignerUri()).isEqualTo(expectedBaseSignerUri);
+ assertThat(client.endpoint()).isEqualTo(expectedEndpoint);
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ public static Stream legacySignerProperties() {
+ return Stream.of(
+ // Only legacy properties
+ Arguments.of(
+ Map.of(
+ CatalogProperties.URI,
+ "https://catalog.com",
+ S3V4RestSignerClient.S3_SIGNER_URI,
+ "https://legacy-signer.com",
+ S3V4RestSignerClient.S3_SIGNER_ENDPOINT,
+ "v1/legacy/sign"),
+ "https://legacy-signer.com",
+ "https://legacy-signer.com/v1/legacy/sign"),
+ // Only new properties
+ Arguments.of(
+ Map.of(
+ CatalogProperties.URI,
+ "https://catalog.com",
+ RESTCatalogProperties.SIGNER_URI,
+ "https://new-signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/new/sign"),
+ "https://new-signer.com",
+ "https://new-signer.com/v1/new/sign"),
+ // Mixed properties: legacy properties take precedence
+ Arguments.of(
+ Map.of(
+ CatalogProperties.URI,
+ "https://catalog.com",
+ RESTCatalogProperties.SIGNER_URI,
+ "https://new-signer.com",
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ "v1/new/sign",
+ S3V4RestSignerClient.S3_SIGNER_URI,
+ "https://legacy-signer.com",
+ S3V4RestSignerClient.S3_SIGNER_ENDPOINT,
+ "v1/legacy/sign"),
+ "https://legacy-signer.com",
+ "https://legacy-signer.com/v1/legacy/sign"),
+ // No signer properties: the catalog URI and the deprecated default endpoint are used
+ Arguments.of(
+ Map.of(CatalogProperties.URI, "https://catalog.com"),
+ "https://catalog.com",
+ "https://catalog.com/" + S3V4RestSignerClient.S3_SIGNER_DEFAULT_ENDPOINT));
+ }
}
diff --git a/azure-bundle/LICENSE b/azure-bundle/LICENSE
index e8c049f4c33b..b0964f5e65ba 100644
--- a/azure-bundle/LICENSE
+++ b/azure-bundle/LICENSE
@@ -207,8 +207,7 @@ This product bundles Azure SDK for Java.
Project URL: https://github.com/Azure/azure-sdk-for-java
License: MIT
-| The MIT License (MIT)
-|
+
| Copyright (c) 2015 Microsoft
|
| Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -238,6 +237,91 @@ License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2
--------------------------------------------------------------------------------
+This product bundles FastDoubleParser (via Jackson JSON Processor).
+
+Copyright: 2023 Werner Randelshofer, Switzerland
+Project URL: https://github.com/wrandelshofer/FastDoubleParser
+License: MIT
+
+| Copyright (c) 2023 Werner Randelshofer, Switzerland
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles fast_float (bundled by FastDoubleParser).
+
+Copyright: 2021 The fast_float authors
+Project URL: https://github.com/fastfloat/fast_float
+License: MIT
+
+| Copyright (c) 2021 The fast_float authors
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles bigint (bundled by FastDoubleParser).
+
+Copyright: 2022 Tim Buktu
+Project URL: https://github.com/tbuktu/bigint
+License: BSD 2-Clause
+
+| Copyright 2022 Tim Buktu
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions
+| are met:
+|
+| 1. Redistributions of source code must retain the above copyright notice, this
+| list of conditions and the following disclaimer.
+|
+| 2. Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+| FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
@@ -245,27 +329,53 @@ This product bundles Microsoft Authentication Library for Java.
Project URL: https://github.com/AzureAD/microsoft-authentication-library-for-java
License: MIT
-| MIT License
-|
-| Copyright (c) Microsoft Corporation. All rights reserved.
-|
-| Permission is hereby granted, free of charge, to any person obtaining a copy
-| of this software and associated documentation files (the "Software"), to deal
-| in the Software without restriction, including without limitation the rights
-| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-| copies of the Software, and to permit persons to whom the Software is
-| furnished to do so, subject to the following conditions:
-|
-| The above copyright notice and this permission notice shall be included in all
-| copies or substantial portions of the Software.
-|
-| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-| SOFTWARE
+
+| Copyright (c) Microsoft Corporation. All rights reserved.
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE
+
+--------------------------------------------------------------------------------
+
+This product bundles MSAL4J Persistence Extension.
+
+Project URL: https://github.com/AzureAD/microsoft-authentication-library-for-java
+License: MIT
+
+| Copyright (c) Microsoft Corporation. All rights reserved.
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE
--------------------------------------------------------------------------------
@@ -276,6 +386,13 @@ License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2
--------------------------------------------------------------------------------
+This product bundles Apache Tomcat Native (netty-tcnative-classes and netty-tcnative-boringssl-static, bundled by Reactor Netty).
+
+Project URL: https://tomcat.apache.org/native-doc/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
This product bundles Reactor Core.
Project URL: https://github.com/reactor/reactor-core
@@ -290,9 +407,16 @@ License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2
--------------------------------------------------------------------------------
-This product bundles Reactor AddOns.
+This product bundles Reactor Pool (bundled by Reactor Netty).
+
+Project URL: https://github.com/reactor/reactor-pool
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product bundles Aalto XML (bundled by Azure SDK for Java).
-Project URL: https://github.com/reactor/reactor-addons
+Project URL: https://github.com/FasterXML/aalto-xml
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
@@ -307,9 +431,8 @@ License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2
This product bundles Reactive Streams.
Project URL: http://www.reactive-streams.org/
-License: MIT
-| MIT No Attribution
-|
+License: MIT-0
+
| Copyright 2014 Reactive Streams
|
| Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
diff --git a/azure-bundle/NOTICE b/azure-bundle/NOTICE
index 07f87c0bc05c..12325baf97d2 100644
--- a/azure-bundle/NOTICE
+++ b/azure-bundle/NOTICE
@@ -8,7 +8,6 @@ The Apache Software Foundation (http://www.apache.org/).
--------------------------------------------------------------------------------
This product bundles Jackson JSON Processor with the following in its NOTICE file:
-|
| # Jackson JSON processor
|
| Jackson is a high-performance, Free/Open Source JSON processing library.
@@ -16,6 +15,10 @@ This product bundles Jackson JSON Processor with the following in its NOTICE fil
| been in development since 2007.
| It is currently developed by a community of developers.
|
+| ## Copyright
+|
+| Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi)
+|
| ## Licensing
|
| Jackson 2.x core and extension components are licensed under Apache License 2.0
@@ -26,7 +29,17 @@ This product bundles Jackson JSON Processor with the following in its NOTICE fil
| A list of contributors may be found from CREDITS(-2.x) file, which is included
| in some artifacts (usually source distributions); but is always available
| from the source code management (SCM) system project uses.
-|
+|
+| ## FastDoubleParser
+|
+| jackson-core bundles a shaded copy of FastDoubleParser .
+| That code is available under an MIT license
+| under the following copyright.
+|
+| Copyright © 2023 Werner Randelshofer, Switzerland. MIT License.
+|
+| See FastDoubleParser-NOTICE for details of other source code included in FastDoubleParser
+| and the licenses and copyrights that apply to that code.
--------------------------------------------------------------------------------
diff --git a/azure-bundle/build.gradle b/azure-bundle/build.gradle
index 0bdc30fdaa7e..fde8adbfc539 100644
--- a/azure-bundle/build.gradle
+++ b/azure-bundle/build.gradle
@@ -23,6 +23,12 @@ project(":iceberg-azure-bundle") {
tasks.jar.dependsOn tasks.shadowJar
+ configurations {
+ implementation {
+ exclude group: 'org.slf4j'
+ }
+ }
+
dependencies {
implementation platform(libs.azuresdk.bom)
implementation "com.azure:azure-storage-file-datalake"
@@ -40,10 +46,6 @@ project(":iceberg-azure-bundle") {
include 'NOTICE'
}
- dependencies {
- exclude(dependency('org.slf4j:slf4j-api'))
- }
-
// relocate Azure-specific versions
relocate 'io.netty', 'org.apache.iceberg.azure.shaded.io.netty'
relocate 'com.fasterxml.jackson', 'org.apache.iceberg.azure.shaded.com.fasterxml.jackson'
@@ -52,4 +54,6 @@ project(":iceberg-azure-bundle") {
jar {
enabled = false
}
+
+ apply from: "${rootDir}/runtime-deps.gradle"
}
diff --git a/azure-bundle/runtime-deps.txt b/azure-bundle/runtime-deps.txt
new file mode 100644
index 000000000000..2e5198f49842
--- /dev/null
+++ b/azure-bundle/runtime-deps.txt
@@ -0,0 +1,43 @@
+com.azure:azure-core-http-netty:1.16.3
+com.azure:azure-core:1.57.1
+com.azure:azure-identity:1.18.2
+com.azure:azure-json:1.5.1
+com.azure:azure-security-keyvault-keys:4.10.6
+com.azure:azure-storage-blob:12.33.3
+com.azure:azure-storage-common:12.32.2
+com.azure:azure-storage-file-datalake:12.26.3
+com.azure:azure-storage-internal-avro:12.18.2
+com.azure:azure-xml:1.2.1
+com.fasterxml.jackson.core:jackson-annotations:2.18.4
+com.fasterxml.jackson.core:jackson-core:2.18.4.1
+com.fasterxml.jackson.core:jackson-databind:2.18.4
+com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.4
+com.microsoft.azure:msal4j-persistence-extension:1.3.0
+com.microsoft.azure:msal4j:1.23.1
+io.netty:netty-buffer:4.1.130.Final
+io.netty:netty-codec-dns:4.1.128.Final
+io.netty:netty-codec-http2:4.1.130.Final
+io.netty:netty-codec-http:4.1.130.Final
+io.netty:netty-codec-socks:4.1.130.Final
+io.netty:netty-codec:4.1.130.Final
+io.netty:netty-common:4.1.130.Final
+io.netty:netty-handler-proxy:4.1.130.Final
+io.netty:netty-handler:4.1.130.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.128.Final
+io.netty:netty-resolver-dns-native-macos:4.1.128.Final
+io.netty:netty-resolver-dns:4.1.128.Final
+io.netty:netty-resolver:4.1.130.Final
+io.netty:netty-tcnative-boringssl-static:2.0.74.Final
+io.netty:netty-tcnative-classes:2.0.74.Final
+io.netty:netty-transport-classes-epoll:4.1.130.Final
+io.netty:netty-transport-classes-kqueue:4.1.130.Final
+io.netty:netty-transport-native-epoll:4.1.130.Final
+io.netty:netty-transport-native-kqueue:4.1.130.Final
+io.netty:netty-transport-native-unix-common:4.1.130.Final
+io.netty:netty-transport:4.1.130.Final
+io.projectreactor.netty:reactor-netty-core:1.2.13
+io.projectreactor.netty:reactor-netty-http:1.2.13
+io.projectreactor:reactor-core:3.7.14
+net.java.dev.jna:jna-platform:5.17.0
+net.java.dev.jna:jna:5.17.0
+org.reactivestreams:reactive-streams:1.0.4
diff --git a/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSFileIO.java b/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSFileIO.java
index 5e343782ab1c..621813a25574 100644
--- a/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSFileIO.java
+++ b/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSFileIO.java
@@ -20,6 +20,7 @@
import static org.apache.iceberg.azure.AzureProperties.ADLS_SAS_TOKEN_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
@@ -31,6 +32,7 @@
import com.azure.core.http.rest.PagedIterable;
import com.azure.core.http.rest.Response;
+import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.file.datalake.DataLakeFileClient;
import com.azure.storage.file.datalake.DataLakeFileSystemClient;
import com.azure.storage.file.datalake.DataLakeFileSystemClientBuilder;
@@ -42,6 +44,7 @@
import java.util.Iterator;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.azure.AzureProperties;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.FileInfo;
import org.apache.iceberg.io.InputFile;
@@ -78,6 +81,23 @@ public void testFileOperations() throws IOException {
assertThat(fileClient.exists()).isFalse();
}
+ @Test
+ public void readMissingLocation() {
+ String path = "path/to/file";
+ String location = AZURITE_CONTAINER.location(path);
+ ADLSFileIO io = createFileIO();
+ DataLakeFileClient fileClient = AZURITE_CONTAINER.fileClient(path);
+ assertThat(fileClient.exists()).isFalse();
+
+ InputFile inputFile = io.newInputFile(location);
+
+ assertThatThrownBy(inputFile::newStream)
+ .isInstanceOf(NotFoundException.class)
+ .hasCauseInstanceOf(BlobStorageException.class)
+ .hasMessage(
+ "Location does not exist: abfs://container@account.dfs.core.windows.net/path/to/file");
+ }
+
@Test
public void testBulkDeleteFiles() {
String path1 = "path/to/file1";
diff --git a/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java b/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
index 8464e57516ce..1edf48eaec35 100644
--- a/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
+++ b/azure/src/integration/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
@@ -39,6 +39,10 @@ public class TestADLSInputStream extends AzuriteTestBase {
private final Random random = new Random(1);
private final AzureProperties azureProperties = new AzureProperties();
+ private String location() {
+ return AZURITE_CONTAINER.location(FILE_PATH);
+ }
+
private DataLakeFileClient fileClient() {
return AZURITE_CONTAINER.fileClient(FILE_PATH);
}
@@ -55,7 +59,8 @@ public void testRead() throws Exception {
setupData(data);
try (SeekableInputStream in =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
int readSize = 1024;
readAndCheck(in, in.getPos(), readSize, data, false);
@@ -90,7 +95,8 @@ public void testReadSingle() throws Exception {
setupData(data);
try (SeekableInputStream in =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
assertThat(in.read()).isEqualTo(i0);
assertThat(in.read()).isEqualTo(i1);
}
@@ -131,7 +137,8 @@ public void testRangeRead() throws Exception {
setupData(expected);
try (RangeReadable in =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
// first 1k
position = 0;
offset = 0;
@@ -164,7 +171,8 @@ private void readAndCheckRanges(
public void testClose() throws Exception {
setupData(randomData(2));
SeekableInputStream closed =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics());
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics());
closed.close();
assertThatThrownBy(() -> closed.seek(0))
.isInstanceOf(IllegalStateException.class)
@@ -178,7 +186,8 @@ public void testSeek() throws Exception {
setupData(data);
try (SeekableInputStream in =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics())) {
in.seek(data.length / 2);
byte[] actual = new byte[data.length / 2];
@@ -193,7 +202,8 @@ public void testSeek() throws Exception {
public void testSeekNegative() throws Exception {
setupData(randomData(2));
SeekableInputStream in =
- new ADLSInputStream(fileClient(), null, azureProperties, MetricsContext.nullMetrics());
+ new ADLSInputStream(
+ location(), fileClient(), null, azureProperties, MetricsContext.nullMetrics());
assertThatThrownBy(() -> in.seek(-3))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot seek: position -3 is negative");
diff --git a/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java b/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
index 73e99e029221..383bec30111b 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
@@ -21,7 +21,6 @@
import com.azure.core.credential.AccessToken;
import com.azure.core.credential.TokenCredential;
import com.azure.core.credential.TokenRequestContext;
-import com.azure.security.keyvault.keys.cryptography.models.KeyWrapAlgorithm;
import com.azure.storage.common.StorageSharedKeyCredential;
import com.azure.storage.file.datalake.DataLakeFileSystemClientBuilder;
import java.io.Serializable;
@@ -53,6 +52,9 @@ public class AzureProperties implements Serializable {
public static final String AZURE_KEYVAULT_KEY_WRAP_ALGORITHM =
"azure.keyvault.key-wrap-algorithm";
+ // Must match KeyWrapAlgorithm.RSA_OAEP_256.getValue() from azure-security-keyvault-keys
+ private static final String DEFAULT_KEY_WRAP_ALGORITHM = "RSA-OAEP-256";
+
/**
* Configure the ADLS token credential provider used to get {@link TokenCredential}. A fully
* qualified concrete class with package that implements the {@link AdlsTokenCredentialProvider}
@@ -136,8 +138,7 @@ public AzureProperties(Map properties) {
this.keyWrapAlgorithm =
properties.getOrDefault(
- AzureProperties.AZURE_KEYVAULT_KEY_WRAP_ALGORITHM,
- KeyWrapAlgorithm.RSA_OAEP_256.getValue());
+ AzureProperties.AZURE_KEYVAULT_KEY_WRAP_ALGORITHM, DEFAULT_KEY_WRAP_ALGORITHM);
}
public Optional adlsReadBlockSize() {
@@ -204,8 +205,8 @@ public Mono getToken(TokenRequestContext request) {
}
}
- public KeyWrapAlgorithm keyWrapAlgorithm() {
- return KeyWrapAlgorithm.fromString(this.keyWrapAlgorithm);
+ public String keyWrapAlgorithm() {
+ return this.keyWrapAlgorithm;
}
public Optional keyVaultUrl() {
diff --git a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputFile.java b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputFile.java
index 95e57bf04d32..5b07534f1368 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputFile.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputFile.java
@@ -55,6 +55,6 @@ public long getLength() {
@Override
public SeekableInputStream newStream() {
- return new ADLSInputStream(fileClient(), fileSize, azureProperties(), metrics());
+ return new ADLSInputStream(location(), fileClient(), fileSize, azureProperties(), metrics());
}
}
diff --git a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
index 55ecade4486a..b1a2d3abfa32 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSInputStream.java
@@ -18,14 +18,18 @@
*/
package org.apache.iceberg.azure.adlsv2;
+import com.azure.storage.blob.models.BlobErrorCode;
+import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.file.datalake.DataLakeFileClient;
import com.azure.storage.file.datalake.models.DataLakeFileOpenInputStreamResult;
+import com.azure.storage.file.datalake.models.DataLakeStorageException;
import com.azure.storage.file.datalake.models.FileRange;
import com.azure.storage.file.datalake.options.DataLakeFileInputStreamOptions;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import org.apache.iceberg.azure.AzureProperties;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.io.FileIOMetricsContext;
import org.apache.iceberg.io.IOUtil;
import org.apache.iceberg.io.RangeReadable;
@@ -46,6 +50,7 @@ class ADLSInputStream extends SeekableInputStream implements RangeReadable {
private static final int SKIP_SIZE = 1024 * 1024;
private final StackTraceElement[] createStack;
+ private final String location;
private final DataLakeFileClient fileClient;
private Long fileSize;
private final AzureProperties azureProperties;
@@ -59,10 +64,12 @@ class ADLSInputStream extends SeekableInputStream implements RangeReadable {
private final Counter readOperations;
ADLSInputStream(
+ String location,
DataLakeFileClient fileClient,
Long fileSize,
AzureProperties azureProperties,
MetricsContext metrics) {
+ this.location = location;
this.fileClient = fileClient;
this.fileSize = fileSize;
this.azureProperties = azureProperties;
@@ -184,6 +191,7 @@ private DataLakeFileOpenInputStreamResult openRange(FileRange range) {
try {
return fileClient.openInputStream(getInputOptions(range));
} catch (RuntimeException e) {
+ throwNotFoundIfNotPresent(e, location);
LOG.error(
"Failed to open input stream for file {}, range {}", fileClient.getFilePath(), range, e);
throw e;
@@ -209,4 +217,20 @@ protected void finalize() throws Throwable {
LOG.warn("Unclosed input stream created by:\n\t{}", trace);
}
}
+
+ private static void throwNotFoundIfNotPresent(Throwable throwable, String location) {
+ if (isFileNotFoundException(throwable)) {
+ throw new NotFoundException(throwable, "Location does not exist: %s", location);
+ }
+ }
+
+ private static boolean isFileNotFoundException(Throwable exception) {
+ if (exception instanceof BlobStorageException blobStorageException) {
+ return BlobErrorCode.BLOB_NOT_FOUND.equals(blobStorageException.getErrorCode());
+ }
+ if (exception instanceof DataLakeStorageException dataLakeStorageException) {
+ return "PathNotFound".equals(dataLakeStorageException.getErrorCode());
+ }
+ return false;
+ }
}
diff --git a/azure/src/main/java/org/apache/iceberg/azure/keymanagement/AzureKeyManagementClient.java b/azure/src/main/java/org/apache/iceberg/azure/keymanagement/AzureKeyManagementClient.java
index 66bf0678bce9..498c432212c5 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/keymanagement/AzureKeyManagementClient.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/keymanagement/AzureKeyManagementClient.java
@@ -80,7 +80,8 @@ private ClientState state() {
keyClientBuilder
.credential(AdlsTokenCredentialProviders.from(allProperties).credential())
.buildClient();
- KeyWrapAlgorithm keyWrapAlgorithm = azureProperties.keyWrapAlgorithm();
+ KeyWrapAlgorithm keyWrapAlgorithm =
+ KeyWrapAlgorithm.fromString(azureProperties.keyWrapAlgorithm());
state = new ClientState(keyClient, keyWrapAlgorithm);
}
}
diff --git a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
index 058bf0372b05..e98061846a88 100644
--- a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
+++ b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/TestADLSInputStream.java
@@ -46,7 +46,13 @@ void before() {
InternalDataLakeFileOpenInputStreamResult openInputStreamResult =
new InternalDataLakeFileOpenInputStreamResult(inputStream, mock());
when(fileClient.openInputStream(any())).thenReturn(openInputStreamResult);
- adlsInputStream = new ADLSInputStream(fileClient, 0L, mock(), mock());
+ adlsInputStream =
+ new ADLSInputStream(
+ "abfs://container@account.dfs.core.windows.net/path/to/file",
+ fileClient,
+ 0L,
+ mock(),
+ mock());
}
@Test
diff --git a/baseline.gradle b/baseline.gradle
index 4efbd89eda02..6b180effbbbf 100644
--- a/baseline.gradle
+++ b/baseline.gradle
@@ -157,6 +157,8 @@ subprojects {
'-Xep:Slf4jThrowable:ERROR',
// Added because it errors out compile, but we need to figure out if we want it
'-Xep:StrictUnusedVariable:OFF',
+ // This rule doesn't enforce the use of method references. That's handled by checkstyle.
+ '-Xep:StringCaseLocaleUsage:ERROR',
// Enforce safe string splitting
'-Xep:StringSplitter:ERROR',
'-Xep:TypeParameterShadowing:OFF',
diff --git a/bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java b/bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java
index cdeaa1ef1e63..23441d0db184 100644
--- a/bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java
+++ b/bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java
@@ -24,6 +24,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.CatalogProperties;
@@ -169,6 +170,18 @@ public void testRenameTableMissingSourceTable() {
super.testRenameTableMissingSourceTable();
}
+ @Disabled("BigQuery Metastore does not support rename tables")
+ @Test
+ public void createTableInUniqueLocation() {
+ super.createTableInUniqueLocation();
+ }
+
+ @Disabled("BigQuery Metastore does not support rename tables")
+ @Test
+ public void dropAfterRenameDoesntCorruptTable() throws IOException {
+ super.dropAfterRenameDoesntCorruptTable();
+ }
+
@Test
public void testIsValidIdentifierWithValidSingleLevelNamespace() {
assertThat(catalog.isValidIdentifier(TableIdentifier.of("dataset1", "table1"))).isTrue();
diff --git a/build.gradle b/build.gradle
index 1369ea8b7e23..ccfc5abee0cf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -38,7 +38,7 @@ buildscript {
classpath 'org.revapi:gradle-revapi:1.8.0'
classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.5.7'
classpath 'com.palantir.gradle.gitversion:gradle-git-version:4.3.0'
- classpath 'org.openapitools:openapi-generator-gradle-plugin:7.20.0'
+ classpath 'org.openapitools:openapi-generator-gradle-plugin:7.22.0'
}
}
@@ -122,6 +122,15 @@ allprojects {
}
}
+tasks.register('checkAllRuntimeDeps') {
+ description = 'Validates runtime dependency baselines for all subprojects that have them'
+ group = 'verification'
+
+ dependsOn subprojects.collect { subproject ->
+ subproject.tasks.matching { it.name == 'checkRuntimeDeps' }
+ }
+}
+
subprojects {
if (it.name == 'iceberg-bom') {
// the BOM does not build anything, the code below expects "source code"
@@ -327,6 +336,7 @@ project(':iceberg-api') {
testImplementation libs.avro.avro
testImplementation libs.esotericsoftware.kryo
testImplementation libs.awaitility
+ testImplementation libs.junit.pioneer
}
tasks.processTestResources.dependsOn rootProject.tasks.buildInfo
@@ -389,7 +399,8 @@ project(':iceberg-core') {
testImplementation libs.jetty.servlet
testImplementation libs.jakarta.servlet
- testImplementation libs.jetty.server
+ testImplementation libs.jetty.compression.server
+ testImplementation libs.jetty.compression.gzip
testImplementation libs.mockserver.netty
testImplementation libs.mockserver.client.java
testImplementation libs.sqlite.jdbc
@@ -401,6 +412,13 @@ project(':iceberg-core') {
exclude group: 'junit'
}
testImplementation libs.awaitility
+
+ // Lock BouncyCastle versions to avoid version mismatches
+ // when these dependencies are added transitively.
+ // Required for TLS tests with MockServer.
+ testImplementation libs.bouncycastle.bcpkix
+ testImplementation libs.bouncycastle.bcutil
+ testImplementation libs.bouncycastle.bcprov
}
}
@@ -446,6 +464,8 @@ project(':iceberg-data') {
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
+ testImplementation(testFixtures(project(':iceberg-parquet')))
}
test {
@@ -466,8 +486,8 @@ project(':iceberg-aliyun') {
implementation project(':iceberg-common')
compileOnly libs.aliyun.sdk.oss
- implementation libs.aliyun.credentials.java
- implementation libs.aliyun.tea
+ compileOnly libs.aliyun.credentials.java
+ compileOnly libs.aliyun.tea
compileOnly libs.jaxb.api
compileOnly libs.activation
compileOnly libs.jaxb.runtime
@@ -543,6 +563,8 @@ project(':iceberg-aws') {
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation libs.awaitility
testImplementation libs.jetty.servlet
+ testImplementation libs.jetty.compression.server
+ testImplementation libs.jetty.compression.gzip
testImplementation libs.mockito.junit.jupiter
}
@@ -567,7 +589,8 @@ project(':iceberg-aws') {
jvmArgs += project.property('extraJvmArgs')
}
- def s3SignerSpec = "$projectDir/src/main/resources/s3-signer-open-api.yaml"
+ // TODO delete once s3-signer-open-api.yaml is removed
+ def s3SignerSpec = layout.projectDirectory.file("src/main/resources/s3-signer-open-api.yaml")
tasks.register('validateS3SignerSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
inputSpec.set(s3SignerSpec)
recommend.set(true)
@@ -713,10 +736,10 @@ project(':iceberg-bigquery') {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
- implementation platform(libs.google.libraries.bom)
+ compileOnly platform(libs.google.libraries.bom)
compileOnly "com.google.cloud:google-cloud-storage"
- implementation "com.google.cloud:google-cloud-bigquery"
- implementation "com.google.cloud:google-cloud-core"
+ compileOnly "com.google.cloud:google-cloud-bigquery"
+ compileOnly "com.google.cloud:google-cloud-core"
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
@@ -925,6 +948,13 @@ project(':iceberg-parquet') {
exclude group: 'org.apache.avro', module: 'avro'
}
+ testFixturesApi(libs.parquet.hadoop) {
+ exclude group: 'org.apache.avro', module: 'avro'
+ // already shaded by Parquet
+ exclude group: 'it.unimi.dsi'
+ exclude group: 'org.codehaus.jackson'
+ }
+
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-data')
@@ -1131,7 +1161,8 @@ project(':iceberg-open-api') {
testFixturesImplementation libs.slf4j.simple
testFixturesImplementation libs.jetty.servlet
- testFixturesImplementation libs.jetty.server
+ testFixturesImplementation libs.jetty.compression.server
+ testFixturesImplementation libs.jetty.compression.gzip
testFixturesImplementation libs.sqlite.jdbc
testFixturesCompileOnly libs.apiguardian
@@ -1156,7 +1187,7 @@ project(':iceberg-open-api') {
.collectEntries { k, v -> { [(k):v, (k.replaceFirst("rck.", "")):v] }} // strip prefix
}
- def restCatalogSpec = "$projectDir/rest-catalog-open-api.yaml"
+ def restCatalogSpec = layout.projectDirectory.file("rest-catalog-open-api.yaml")
tasks.register('validateRESTCatalogSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
inputSpec.set(restCatalogSpec)
recommend.set(true)
diff --git a/core/src/main/java/org/apache/iceberg/stats/BaseContentStats.java b/core/src/main/java/org/apache/iceberg/BaseContentStats.java
similarity index 98%
rename from core/src/main/java/org/apache/iceberg/stats/BaseContentStats.java
rename to core/src/main/java/org/apache/iceberg/BaseContentStats.java
index be56c411b6a7..45900b03e299 100644
--- a/core/src/main/java/org/apache/iceberg/stats/BaseContentStats.java
+++ b/core/src/main/java/org/apache/iceberg/BaseContentStats.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
import java.io.Serializable;
import java.util.List;
@@ -24,7 +24,6 @@
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -33,14 +32,14 @@
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
-public class BaseContentStats implements ContentStats, Serializable {
+class BaseContentStats implements ContentStats, Serializable {
private final List> fieldStats;
private final Map> fieldStatsById;
private final Types.StructType statsStruct;
/** Used by Avro reflection to instantiate this class when reading manifest files. */
- public BaseContentStats(Types.StructType projection) {
+ BaseContentStats(Types.StructType projection) {
this.statsStruct = projection;
this.fieldStats = Lists.newArrayListWithCapacity(projection.fields().size());
this.fieldStatsById = Maps.newLinkedHashMapWithExpectedSize(projection.fields().size());
diff --git a/core/src/main/java/org/apache/iceberg/stats/BaseFieldStats.java b/core/src/main/java/org/apache/iceberg/BaseFieldStats.java
similarity index 98%
rename from core/src/main/java/org/apache/iceberg/stats/BaseFieldStats.java
rename to core/src/main/java/org/apache/iceberg/BaseFieldStats.java
index 470303179bf5..11da570b8faa 100644
--- a/core/src/main/java/org/apache/iceberg/stats/BaseFieldStats.java
+++ b/core/src/main/java/org/apache/iceberg/BaseFieldStats.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
@@ -28,7 +28,7 @@
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.ByteBuffers;
-public class BaseFieldStats extends SupportsIndexProjection implements FieldStats {
+class BaseFieldStats extends SupportsIndexProjection implements FieldStats {
private static final int[] IDENTITY_MAPPING = identityMapping();
private final int fieldId;
private final Type type;
diff --git a/core/src/main/java/org/apache/iceberg/BaseFile.java b/core/src/main/java/org/apache/iceberg/BaseFile.java
index a02e0eff55a2..3c31c50f099f 100644
--- a/core/src/main/java/org/apache/iceberg/BaseFile.java
+++ b/core/src/main/java/org/apache/iceberg/BaseFile.java
@@ -45,7 +45,7 @@ abstract class BaseFile extends SupportsIndexProjection
StructLike,
SpecificData.SchemaConstructable,
Serializable {
- private static final FileContent[] FILE_CONTENT_VALUES = FileContent.values();
+
static final Types.StructType EMPTY_STRUCT_TYPE = Types.StructType.of();
static final PartitionData EMPTY_PARTITION_DATA =
new PartitionData(EMPTY_STRUCT_TYPE) {
@@ -316,7 +316,7 @@ public void put(int i, Object value) {
protected void internalSet(int pos, T value) {
switch (pos) {
case 0:
- this.content = value != null ? FILE_CONTENT_VALUES[(Integer) value] : FileContent.DATA;
+ this.content = value != null ? FileContent.fromId((Integer) value) : FileContent.DATA;
return;
case 1:
// always coerce to String for Serializable
diff --git a/core/src/main/java/org/apache/iceberg/CatalogProperties.java b/core/src/main/java/org/apache/iceberg/CatalogProperties.java
index 59744e50924f..6b85ccbc87bc 100644
--- a/core/src/main/java/org/apache/iceberg/CatalogProperties.java
+++ b/core/src/main/java/org/apache/iceberg/CatalogProperties.java
@@ -158,6 +158,15 @@ private CatalogProperties() {}
public static final String APP_NAME = "app-name";
public static final String USER = "user";
+ /**
+ * Requests that the catalog provide unique locations for new tables.
+ *
+ * Relevant only for catalogs which support unique table locations.
+ */
+ public static final String UNIQUE_TABLE_LOCATION = "unique-table-location";
+
+ public static final boolean UNIQUE_TABLE_LOCATION_DEFAULT = false;
+
public static final String AUTH_SESSION_TIMEOUT_MS = "auth.session-timeout-ms";
public static final long AUTH_SESSION_TIMEOUT_MS_DEFAULT = TimeUnit.HOURS.toMillis(1);
diff --git a/api/src/main/java/org/apache/iceberg/stats/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java
similarity index 92%
rename from api/src/main/java/org/apache/iceberg/stats/ContentStats.java
rename to core/src/main/java/org/apache/iceberg/ContentStats.java
index b39db2565163..623a8eb39baf 100644
--- a/api/src/main/java/org/apache/iceberg/stats/ContentStats.java
+++ b/core/src/main/java/org/apache/iceberg/ContentStats.java
@@ -16,13 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
import java.util.List;
-import org.apache.iceberg.StructLike;
import org.apache.iceberg.types.Types;
-public interface ContentStats extends StructLike {
+interface ContentStats extends StructLike {
/** A list of all the {@link FieldStats} */
List> fieldStats();
diff --git a/core/src/main/java/org/apache/iceberg/DeletionVector.java b/core/src/main/java/org/apache/iceberg/DeletionVector.java
new file mode 100644
index 000000000000..0fc8f259f075
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/DeletionVector.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import org.apache.iceberg.types.Types;
+
+/**
+ * Metadata about a deletion vector.
+ *
+ * Tracks where a DV blob can be read. The DV blob follows the format defined by the
+ * deletion-vector-v1 blob type in the Puffin spec.
+ */
+interface DeletionVector {
+ Types.NestedField LOCATION =
+ Types.NestedField.required(
+ 155, "location", Types.StringType.get(), "Location of the file containing the DV");
+ Types.NestedField OFFSET =
+ Types.NestedField.required(
+ 144, "offset", Types.LongType.get(), "Offset in the file where the DV content starts");
+ Types.NestedField SIZE_IN_BYTES =
+ Types.NestedField.required(
+ 145,
+ "size_in_bytes",
+ Types.LongType.get(),
+ "Length of the referenced DV content stored in the file");
+ Types.NestedField CARDINALITY =
+ Types.NestedField.required(
+ 156,
+ "cardinality",
+ Types.LongType.get(),
+ "Number of set bits (deleted rows) in the vector");
+
+ static Types.StructType schema() {
+ return Types.StructType.of(LOCATION, OFFSET, SIZE_IN_BYTES, CARDINALITY);
+ }
+
+ /** Returns the location of the file containing the deletion vector. */
+ String location();
+
+ /** Returns the offset in the file where the deletion vector content starts. */
+ long offset();
+
+ /** Returns the size in bytes of the deletion vector content. */
+ long sizeInBytes();
+
+ /** Returns the number of set bits (deleted rows) in the vector. */
+ long cardinality();
+
+ /** Copies this deletion vector. */
+ DeletionVector copy();
+}
diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
new file mode 100644
index 000000000000..0eb7c2fe1eb6
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.io.Serializable;
+import org.apache.iceberg.avro.SupportsIndexProjection;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types;
+
+/** Mutable {@link StructLike} implementation of {@link DeletionVector}. */
+class DeletionVectorStruct extends SupportsIndexProjection implements DeletionVector, Serializable {
+ private static final Types.StructType BASE_TYPE =
+ Types.StructType.of(
+ DeletionVector.LOCATION,
+ DeletionVector.OFFSET,
+ DeletionVector.SIZE_IN_BYTES,
+ DeletionVector.CARDINALITY);
+
+ private String location = null;
+ private long offset = -1L;
+ private long sizeInBytes = -1L;
+ private long cardinality = -1L;
+
+ DeletionVectorStruct(Types.StructType type) {
+ super(BASE_TYPE, type);
+ }
+
+ private DeletionVectorStruct(DeletionVectorStruct toCopy) {
+ super(toCopy);
+ this.location = toCopy.location;
+ this.offset = toCopy.offset;
+ this.sizeInBytes = toCopy.sizeInBytes;
+ this.cardinality = toCopy.cardinality;
+ }
+
+ private DeletionVectorStruct(String location, long offset, long sizeInBytes, long cardinality) {
+ super(BASE_TYPE, BASE_TYPE);
+ this.location = location;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ this.cardinality = cardinality;
+ }
+
+ @Override
+ public String location() {
+ return location;
+ }
+
+ @Override
+ public long offset() {
+ return offset;
+ }
+
+ @Override
+ public long sizeInBytes() {
+ return sizeInBytes;
+ }
+
+ @Override
+ public long cardinality() {
+ return cardinality;
+ }
+
+ @Override
+ public DeletionVectorStruct copy() {
+ return new DeletionVectorStruct(this);
+ }
+
+ @Override
+ protected T internalGet(int pos, Class javaClass) {
+ return javaClass.cast(getByPos(pos));
+ }
+
+ private Object getByPos(int pos) {
+ switch (pos) {
+ case 0:
+ return location;
+ case 1:
+ return offset;
+ case 2:
+ return sizeInBytes;
+ case 3:
+ return cardinality;
+ default:
+ throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
+ }
+ }
+
+ @Override
+ protected void internalSet(int pos, T value) {
+ switch (pos) {
+ case 0:
+ // always coerce to String for Serializable
+ this.location = value.toString();
+ break;
+ case 1:
+ this.offset = (Long) value;
+ break;
+ case 2:
+ this.sizeInBytes = (Long) value;
+ break;
+ case 3:
+ this.cardinality = (Long) value;
+ break;
+ default:
+ // ignore the object, it must be from a newer version of the format
+ }
+ }
+
+ static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("location", location)
+ .add("offset", offset)
+ .add("size_in_bytes", sizeInBytes)
+ .add("cardinality", cardinality)
+ .toString();
+ }
+
+ static class Builder {
+ private String location = null;
+ private long offset = -1L;
+ private long sizeInBytes = -1L;
+ private long cardinality = -1L;
+
+ Builder location(String dvLocation) {
+ this.location = dvLocation;
+ return this;
+ }
+
+ Builder offset(long dvOffset) {
+ this.offset = dvOffset;
+ return this;
+ }
+
+ Builder sizeInBytes(long dvSizeInBytes) {
+ this.sizeInBytes = dvSizeInBytes;
+ return this;
+ }
+
+ Builder cardinality(long dvCardinality) {
+ this.cardinality = dvCardinality;
+ return this;
+ }
+
+ DeletionVectorStruct build() {
+ Preconditions.checkArgument(location != null, "Invalid location: null");
+ Preconditions.checkArgument(offset >= 0, "Invalid offset: %s (must be >= 0)", offset);
+ Preconditions.checkArgument(
+ sizeInBytes >= 0, "Invalid size in bytes: %s (must be >= 0)", sizeInBytes);
+ Preconditions.checkArgument(
+ cardinality >= 0, "Invalid cardinality: %s (must be >= 0)", cardinality);
+ return new DeletionVectorStruct(location, offset, sizeInBytes, cardinality);
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/EntryStatus.java b/core/src/main/java/org/apache/iceberg/EntryStatus.java
new file mode 100644
index 000000000000..ceabeb562415
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/EntryStatus.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+/** Status of an entry in a manifest file. */
+enum EntryStatus {
+ EXISTING(0),
+ ADDED(1),
+ DELETED(2),
+ /** Indicates an entry that has been replaced by a column update or DV change. Added in v4. */
+ REPLACED(3);
+
+ private static final EntryStatus[] VALUES = EntryStatus.values();
+
+ private final int id;
+
+ EntryStatus(int id) {
+ this.id = id;
+ }
+
+ public int id() {
+ return id;
+ }
+
+ static EntryStatus fromId(int id) {
+ return VALUES[id];
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java b/core/src/main/java/org/apache/iceberg/FieldStatistic.java
similarity index 93%
rename from api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java
rename to core/src/main/java/org/apache/iceberg/FieldStatistic.java
index 72058e5253ab..85712384254c 100644
--- a/api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java
+++ b/core/src/main/java/org/apache/iceberg/FieldStatistic.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
import static org.apache.iceberg.types.Types.NestedField.optional;
@@ -25,12 +25,12 @@
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
-public enum FieldStatistic {
+enum FieldStatistic {
VALUE_COUNT(1, "value_count"),
NULL_VALUE_COUNT(2, "null_value_count"),
NAN_VALUE_COUNT(3, "nan_value_count"),
- AVG_VALUE_SIZE(4, "avg_value_size"),
- MAX_VALUE_SIZE(5, "max_value_size"),
+ AVG_VALUE_SIZE(4, "avg_value_size_in_bytes"),
+ MAX_VALUE_SIZE(5, "max_value_size_in_bytes"),
LOWER_BOUND(6, "lower_bound"),
UPPER_BOUND(7, "upper_bound"),
EXACT_BOUNDS(8, "exact_bounds");
@@ -125,13 +125,13 @@ public static Types.StructType fieldStatsFor(Types.NestedField field, int baseFi
baseFieldId + AVG_VALUE_SIZE.offset(),
AVG_VALUE_SIZE.fieldName(),
Types.IntegerType.get(),
- "Avg value size of variable-length types (String, Binary)"));
+ "Avg value size in bytes of variable-length types (String, Binary)"));
fields.add(
optional(
baseFieldId + MAX_VALUE_SIZE.offset(),
MAX_VALUE_SIZE.fieldName(),
Types.IntegerType.get(),
- "Max value size of variable-length types (String, Binary)"));
+ "Max value size in bytes of variable-length types (String, Binary)"));
}
fields.add(
diff --git a/api/src/main/java/org/apache/iceberg/stats/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java
similarity index 92%
rename from api/src/main/java/org/apache/iceberg/stats/FieldStats.java
rename to core/src/main/java/org/apache/iceberg/FieldStats.java
index 6411b479af49..e42d774c7cee 100644
--- a/api/src/main/java/org/apache/iceberg/stats/FieldStats.java
+++ b/core/src/main/java/org/apache/iceberg/FieldStats.java
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
-import org.apache.iceberg.StructLike;
import org.apache.iceberg.types.Type;
-public interface FieldStats extends StructLike {
+interface FieldStats extends StructLike {
/** The field ID of the statistic */
int fieldId();
diff --git a/core/src/main/java/org/apache/iceberg/FileCleanupStrategy.java b/core/src/main/java/org/apache/iceberg/FileCleanupStrategy.java
index b55280a6537f..573aef057ff6 100644
--- a/core/src/main/java/org/apache/iceberg/FileCleanupStrategy.java
+++ b/core/src/main/java/org/apache/iceberg/FileCleanupStrategy.java
@@ -26,7 +26,9 @@
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.SupportsBulkOperations;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,13 +80,15 @@ public abstract void cleanFiles(
ExpireSnapshots.CleanupLevel cleanupLevel);
private static final Schema MANIFEST_PROJECTION =
- ManifestFile.schema()
- .select(
- "manifest_path",
- "manifest_length",
- "partition_spec_id",
- "added_snapshot_id",
- "deleted_data_files_count");
+ TypeUtil.select(
+ ManifestFile.schema(),
+ ImmutableSet.of(
+ ManifestFile.PATH.fieldId(),
+ ManifestFile.LENGTH.fieldId(),
+ ManifestFile.SPEC_ID.fieldId(),
+ ManifestFile.SNAPSHOT_ID.fieldId(),
+ ManifestFile.ADDED_FILES_COUNT.fieldId(),
+ ManifestFile.DELETED_FILES_COUNT.fieldId()));
protected CloseableIterable readManifests(Snapshot snapshot) {
if (snapshot.manifestListLocation() != null) {
diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
index b2ce5fa2aa11..f154c982d1c7 100644
--- a/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
+++ b/core/src/main/java/org/apache/iceberg/GenericManifestEntry.java
@@ -26,7 +26,6 @@
class GenericManifestEntry>
implements ManifestEntry, IndexedRecord, SpecificData.SchemaConstructable, StructLike {
- private static final Status[] STATUS_VALUES = Status.values();
private final org.apache.avro.Schema schema;
private Status status = Status.EXISTING;
private Long snapshotId = null;
@@ -159,7 +158,7 @@ public void setFileSequenceNumber(long newFileSequenceNumber) {
public void put(int i, Object v) {
switch (i) {
case 0:
- this.status = STATUS_VALUES[(Integer) v];
+ this.status = Status.fromId((Integer) v);
return;
case 1:
this.snapshotId = (Long) v;
diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java
index ac93222d01b5..9624484ffe0c 100644
--- a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java
+++ b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java
@@ -40,8 +40,6 @@ public class GenericManifestFile extends SupportsIndexProjection
implements ManifestFile, StructLike, IndexedRecord, SchemaConstructable, Serializable {
private static final Schema AVRO_SCHEMA =
AvroSchemaUtil.convert(ManifestFile.schema(), "manifest_file");
- private static final ManifestContent[] MANIFEST_CONTENT_VALUES = ManifestContent.values();
-
private transient Schema avroSchema; // not final for Java serialization
// data fields
@@ -343,7 +341,7 @@ protected void internalSet(int basePos, T value) {
return;
case 3:
this.content =
- value != null ? MANIFEST_CONTENT_VALUES[(Integer) value] : ManifestContent.DATA;
+ value != null ? ManifestContent.fromId((Integer) value) : ManifestContent.DATA;
return;
case 4:
this.sequenceNumber = value != null ? (Long) value : 0;
diff --git a/core/src/main/java/org/apache/iceberg/ManifestEntry.java b/core/src/main/java/org/apache/iceberg/ManifestEntry.java
index 4dce92cf5c2f..635231069ffc 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestEntry.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestEntry.java
@@ -30,6 +30,8 @@ enum Status {
ADDED(1),
DELETED(2);
+ private static final Status[] VALUES = Status.values();
+
private final int id;
Status(int id) {
@@ -39,6 +41,10 @@ enum Status {
public int id() {
return id;
}
+
+ static Status fromId(int id) {
+ return VALUES[id];
+ }
}
// ids for data-file columns are assigned from 1000
diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfo.java b/core/src/main/java/org/apache/iceberg/ManifestInfo.java
new file mode 100644
index 000000000000..e87287911426
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/ManifestInfo.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import org.apache.iceberg.types.Types;
+
+/** Summary information about a manifest referenced by a root manifest entry. */
+interface ManifestInfo {
+ Types.NestedField ADDED_FILES_COUNT =
+ Types.NestedField.required(
+ 504, "added_files_count", Types.IntegerType.get(), "Number of files added");
+ Types.NestedField EXISTING_FILES_COUNT =
+ Types.NestedField.required(
+ 505, "existing_files_count", Types.IntegerType.get(), "Number of existing files");
+ Types.NestedField DELETED_FILES_COUNT =
+ Types.NestedField.required(
+ 506, "deleted_files_count", Types.IntegerType.get(), "Number of deleted files");
+ Types.NestedField REPLACED_FILES_COUNT =
+ Types.NestedField.required(
+ 520, "replaced_files_count", Types.IntegerType.get(), "Number of replaced files");
+ Types.NestedField ADDED_ROWS_COUNT =
+ Types.NestedField.required(
+ 512, "added_rows_count", Types.LongType.get(), "Number of rows in added files");
+ Types.NestedField EXISTING_ROWS_COUNT =
+ Types.NestedField.required(
+ 513, "existing_rows_count", Types.LongType.get(), "Number of rows in existing files");
+ Types.NestedField DELETED_ROWS_COUNT =
+ Types.NestedField.required(
+ 514, "deleted_rows_count", Types.LongType.get(), "Number of rows in deleted files");
+ Types.NestedField REPLACED_ROWS_COUNT =
+ Types.NestedField.required(
+ 521, "replaced_rows_count", Types.LongType.get(), "Number of rows in replaced files");
+ Types.NestedField MIN_SEQUENCE_NUMBER =
+ Types.NestedField.required(
+ 516,
+ "min_sequence_number",
+ Types.LongType.get(),
+ "Minimum sequence number of files in this manifest");
+ Types.NestedField DV =
+ Types.NestedField.optional(
+ 522, "dv", Types.BinaryType.get(), "Deletion vector for manifest entries");
+ Types.NestedField DV_CARDINALITY =
+ Types.NestedField.optional(
+ 523,
+ "dv_cardinality",
+ Types.LongType.get(),
+ "Number of entries marked as deleted in the DV");
+
+ static Types.StructType schema() {
+ return Types.StructType.of(
+ ADDED_FILES_COUNT,
+ EXISTING_FILES_COUNT,
+ DELETED_FILES_COUNT,
+ REPLACED_FILES_COUNT,
+ ADDED_ROWS_COUNT,
+ EXISTING_ROWS_COUNT,
+ DELETED_ROWS_COUNT,
+ REPLACED_ROWS_COUNT,
+ MIN_SEQUENCE_NUMBER,
+ DV,
+ DV_CARDINALITY);
+ }
+
+ /** Returns the number of files added by this manifest. */
+ int addedFilesCount();
+
+ /** Returns the number of existing files referenced by this manifest. */
+ int existingFilesCount();
+
+ /** Returns the number of deleted files in this manifest. */
+ int deletedFilesCount();
+
+ /** Returns the number of replaced files in this manifest. */
+ int replacedFilesCount();
+
+ /** Returns the number of rows in added files. */
+ long addedRowsCount();
+
+ /** Returns the number of rows in existing files. */
+ long existingRowsCount();
+
+ /** Returns the number of rows in deleted files. */
+ long deletedRowsCount();
+
+ /** Returns the number of rows in replaced files. */
+ long replacedRowsCount();
+
+ /** Returns the minimum sequence number of files in this manifest. */
+ long minSequenceNumber();
+
+ /** Returns the deletion vector bitmap, or null if not present. */
+ ByteBuffer dv();
+
+ /** Returns the number of entries marked as deleted in the DV, or null if not present. */
+ Long dvCardinality();
+
+ /** Copies this manifest info. */
+ ManifestInfo copy();
+}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
new file mode 100644
index 000000000000..922047bffedd
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/ManifestInfoStruct.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.iceberg.avro.SupportsIndexProjection;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+
+/** Mutable {@link StructLike} implementation of {@link ManifestInfo}. */
+class ManifestInfoStruct extends SupportsIndexProjection implements ManifestInfo, Serializable {
+ private static final Types.StructType BASE_TYPE =
+ Types.StructType.of(
+ ManifestInfo.ADDED_FILES_COUNT,
+ ManifestInfo.EXISTING_FILES_COUNT,
+ ManifestInfo.DELETED_FILES_COUNT,
+ ManifestInfo.REPLACED_FILES_COUNT,
+ ManifestInfo.ADDED_ROWS_COUNT,
+ ManifestInfo.EXISTING_ROWS_COUNT,
+ ManifestInfo.DELETED_ROWS_COUNT,
+ ManifestInfo.REPLACED_ROWS_COUNT,
+ ManifestInfo.MIN_SEQUENCE_NUMBER,
+ ManifestInfo.DV,
+ ManifestInfo.DV_CARDINALITY);
+
+ private int addedFilesCount = -1;
+ private int existingFilesCount = -1;
+ private int deletedFilesCount = -1;
+ private int replacedFilesCount = -1;
+ private long addedRowsCount = -1L;
+ private long existingRowsCount = -1L;
+ private long deletedRowsCount = -1L;
+ private long replacedRowsCount = -1L;
+ private long minSequenceNumber = -1L;
+ private byte[] dv = null;
+ private Long dvCardinality = null;
+
+ ManifestInfoStruct(Types.StructType type) {
+ super(BASE_TYPE, type);
+ }
+
+ private ManifestInfoStruct(ManifestInfoStruct toCopy) {
+ super(toCopy);
+ this.addedFilesCount = toCopy.addedFilesCount;
+ this.existingFilesCount = toCopy.existingFilesCount;
+ this.deletedFilesCount = toCopy.deletedFilesCount;
+ this.replacedFilesCount = toCopy.replacedFilesCount;
+ this.addedRowsCount = toCopy.addedRowsCount;
+ this.existingRowsCount = toCopy.existingRowsCount;
+ this.deletedRowsCount = toCopy.deletedRowsCount;
+ this.replacedRowsCount = toCopy.replacedRowsCount;
+ this.minSequenceNumber = toCopy.minSequenceNumber;
+ this.dv = toCopy.dv != null ? Arrays.copyOf(toCopy.dv, toCopy.dv.length) : null;
+ this.dvCardinality = toCopy.dvCardinality;
+ }
+
+ private ManifestInfoStruct(
+ int addedFilesCount,
+ int existingFilesCount,
+ int deletedFilesCount,
+ int replacedFilesCount,
+ long addedRowsCount,
+ long existingRowsCount,
+ long deletedRowsCount,
+ long replacedRowsCount,
+ long minSequenceNumber,
+ byte[] dv,
+ Long dvCardinality) {
+ super(BASE_TYPE, BASE_TYPE);
+ this.addedFilesCount = addedFilesCount;
+ this.existingFilesCount = existingFilesCount;
+ this.deletedFilesCount = deletedFilesCount;
+ this.replacedFilesCount = replacedFilesCount;
+ this.addedRowsCount = addedRowsCount;
+ this.existingRowsCount = existingRowsCount;
+ this.deletedRowsCount = deletedRowsCount;
+ this.replacedRowsCount = replacedRowsCount;
+ this.minSequenceNumber = minSequenceNumber;
+ this.dv = dv;
+ this.dvCardinality = dvCardinality;
+ }
+
+ @Override
+ public int addedFilesCount() {
+ return addedFilesCount;
+ }
+
+ @Override
+ public int existingFilesCount() {
+ return existingFilesCount;
+ }
+
+ @Override
+ public int deletedFilesCount() {
+ return deletedFilesCount;
+ }
+
+ @Override
+ public int replacedFilesCount() {
+ return replacedFilesCount;
+ }
+
+ @Override
+ public long addedRowsCount() {
+ return addedRowsCount;
+ }
+
+ @Override
+ public long existingRowsCount() {
+ return existingRowsCount;
+ }
+
+ @Override
+ public long deletedRowsCount() {
+ return deletedRowsCount;
+ }
+
+ @Override
+ public long replacedRowsCount() {
+ return replacedRowsCount;
+ }
+
+ @Override
+ public long minSequenceNumber() {
+ return minSequenceNumber;
+ }
+
+ @Override
+ public ByteBuffer dv() {
+ return dv != null ? ByteBuffer.wrap(dv) : null;
+ }
+
+ @Override
+ public Long dvCardinality() {
+ return dvCardinality;
+ }
+
+ @Override
+ public ManifestInfoStruct copy() {
+ return new ManifestInfoStruct(this);
+ }
+
+ @Override
+ protected T internalGet(int pos, Class javaClass) {
+ return javaClass.cast(getByPos(pos));
+ }
+
+ private Object getByPos(int pos) {
+ switch (pos) {
+ case 0:
+ return addedFilesCount;
+ case 1:
+ return existingFilesCount;
+ case 2:
+ return deletedFilesCount;
+ case 3:
+ return replacedFilesCount;
+ case 4:
+ return addedRowsCount;
+ case 5:
+ return existingRowsCount;
+ case 6:
+ return deletedRowsCount;
+ case 7:
+ return replacedRowsCount;
+ case 8:
+ return minSequenceNumber;
+ case 9:
+ return dv();
+ case 10:
+ return dvCardinality;
+ default:
+ throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
+ }
+ }
+
+ @Override
+ protected void internalSet(int pos, T value) {
+ switch (pos) {
+ case 0:
+ this.addedFilesCount = (Integer) value;
+ break;
+ case 1:
+ this.existingFilesCount = (Integer) value;
+ break;
+ case 2:
+ this.deletedFilesCount = (Integer) value;
+ break;
+ case 3:
+ this.replacedFilesCount = (Integer) value;
+ break;
+ case 4:
+ this.addedRowsCount = (Long) value;
+ break;
+ case 5:
+ this.existingRowsCount = (Long) value;
+ break;
+ case 6:
+ this.deletedRowsCount = (Long) value;
+ break;
+ case 7:
+ this.replacedRowsCount = (Long) value;
+ break;
+ case 8:
+ this.minSequenceNumber = (Long) value;
+ break;
+ case 9:
+ this.dv = ByteBuffers.toByteArray((ByteBuffer) value);
+ break;
+ case 10:
+ this.dvCardinality = (Long) value;
+ break;
+ default:
+ // ignore the object, it must be from a newer version of the format
+ }
+ }
+
+ static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("added_files_count", addedFilesCount)
+ .add("existing_files_count", existingFilesCount)
+ .add("deleted_files_count", deletedFilesCount)
+ .add("replaced_files_count", replacedFilesCount)
+ .add("added_rows_count", addedRowsCount)
+ .add("existing_rows_count", existingRowsCount)
+ .add("deleted_rows_count", deletedRowsCount)
+ .add("replaced_rows_count", replacedRowsCount)
+ .add("min_sequence_number", minSequenceNumber)
+ .add("dv", dv == null ? "null" : "(binary)")
+ .add("dv_cardinality", dvCardinality == null ? "null" : dvCardinality)
+ .toString();
+ }
+
+ static class Builder {
+ private int addedFilesCount = -1;
+ private int existingFilesCount = -1;
+ private int deletedFilesCount = -1;
+ private int replacedFilesCount = -1;
+ private long addedRowsCount = -1L;
+ private long existingRowsCount = -1L;
+ private long deletedRowsCount = -1L;
+ private long replacedRowsCount = -1L;
+ private long minSequenceNumber = -1L;
+ private byte[] dv = null;
+ private Long dvCardinality = null;
+
+ Builder addedFilesCount(int count) {
+ this.addedFilesCount = count;
+ return this;
+ }
+
+ Builder existingFilesCount(int count) {
+ this.existingFilesCount = count;
+ return this;
+ }
+
+ Builder deletedFilesCount(int count) {
+ this.deletedFilesCount = count;
+ return this;
+ }
+
+ Builder replacedFilesCount(int count) {
+ this.replacedFilesCount = count;
+ return this;
+ }
+
+ Builder addedRowsCount(long count) {
+ this.addedRowsCount = count;
+ return this;
+ }
+
+ Builder existingRowsCount(long count) {
+ this.existingRowsCount = count;
+ return this;
+ }
+
+ Builder deletedRowsCount(long count) {
+ this.deletedRowsCount = count;
+ return this;
+ }
+
+ Builder replacedRowsCount(long count) {
+ this.replacedRowsCount = count;
+ return this;
+ }
+
+ Builder minSequenceNumber(long sequenceNumber) {
+ this.minSequenceNumber = sequenceNumber;
+ return this;
+ }
+
+ Builder dv(ByteBuffer buffer) {
+ this.dv = buffer != null ? ByteBuffers.toByteArray(buffer) : null;
+ return this;
+ }
+
+ Builder dv(byte[] buffer) {
+ this.dv = buffer;
+ return this;
+ }
+
+ Builder dvCardinality(Long cardinality) {
+ this.dvCardinality = cardinality;
+ return this;
+ }
+
+ ManifestInfoStruct build() {
+ Preconditions.checkArgument(
+ addedFilesCount >= 0, "Invalid added files count: %s (must be >= 0)", addedFilesCount);
+ Preconditions.checkArgument(
+ existingFilesCount >= 0,
+ "Invalid existing files count: %s (must be >= 0)",
+ existingFilesCount);
+ Preconditions.checkArgument(
+ deletedFilesCount >= 0,
+ "Invalid deleted files count: %s (must be >= 0)",
+ deletedFilesCount);
+ Preconditions.checkArgument(
+ replacedFilesCount >= 0,
+ "Invalid replaced files count: %s (must be >= 0)",
+ replacedFilesCount);
+ Preconditions.checkArgument(
+ addedRowsCount >= 0, "Invalid added rows count: %s (must be >= 0)", addedRowsCount);
+ Preconditions.checkArgument(
+ existingRowsCount >= 0,
+ "Invalid existing rows count: %s (must be >= 0)",
+ existingRowsCount);
+ Preconditions.checkArgument(
+ deletedRowsCount >= 0, "Invalid deleted rows count: %s (must be >= 0)", deletedRowsCount);
+ Preconditions.checkArgument(
+ replacedRowsCount >= 0,
+ "Invalid replaced rows count: %s (must be >= 0)",
+ replacedRowsCount);
+ Preconditions.checkArgument(
+ minSequenceNumber >= 0,
+ "Invalid min sequence number: %s (must be >= 0)",
+ minSequenceNumber);
+ Preconditions.checkArgument(
+ (dv == null) == (dvCardinality == null),
+ "Invalid DV and cardinality: must both be null or non-null");
+ Preconditions.checkArgument(
+ dvCardinality == null || dvCardinality > 0,
+ "Invalid DV cardinality: %s (must be positive)",
+ dvCardinality);
+ return new ManifestInfoStruct(
+ addedFilesCount,
+ existingFilesCount,
+ deletedFilesCount,
+ replacedFilesCount,
+ addedRowsCount,
+ existingRowsCount,
+ deletedRowsCount,
+ replacedRowsCount,
+ minSequenceNumber,
+ dv,
+ dvCardinality);
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/ManifestReader.java b/core/src/main/java/org/apache/iceberg/ManifestReader.java
index 668a3764de1d..09bbe8b0cc6b 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestReader.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestReader.java
@@ -43,6 +43,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.PartitionSet;
import org.slf4j.Logger;
@@ -68,6 +69,11 @@ public class ManifestReader> extends CloseableGroup
"upper_bounds",
"record_count");
+ private static final Schema STATUS_ONLY_PROJECTION =
+ TypeUtil.select(
+ ManifestEntry.getSchema(Types.StructType.of()),
+ ImmutableSet.of(ManifestEntry.STATUS.fieldId()));
+
protected enum FileType {
DATA_FILES(GenericDataFile.class),
DELETE_FILES(GenericDeleteFile.class);
@@ -157,9 +163,7 @@ private static > Map readMetadata(Input
Map metadata;
try {
try (CloseableIterable> headerReader =
- InternalData.read(FileFormat.AVRO, inputFile)
- .project(ManifestEntry.getSchema(Types.StructType.of()).select("status"))
- .build()) {
+ InternalData.read(FileFormat.AVRO, inputFile).project(STATUS_ONLY_PROJECTION).build()) {
if (headerReader instanceof AvroIterable) {
metadata = ((AvroIterable>) headerReader).getMetadata();
diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
index eed4e56dc05a..e072382543b7 100644
--- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
+++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java
@@ -289,7 +289,11 @@ private void addInternal(DeleteFile file) {
protected void validateNewDeleteFile(DeleteFile file) {
Preconditions.checkNotNull(file, "Invalid delete file: null");
- switch (formatVersion()) {
+ validateDeleteFileForVersion(file, formatVersion());
+ }
+
+ private static void validateDeleteFileForVersion(DeleteFile file, int formatVersion) {
+ switch (formatVersion) {
case 1:
throw new IllegalArgumentException("Deletes are supported in V2 and above");
case 2:
@@ -303,11 +307,11 @@ protected void validateNewDeleteFile(DeleteFile file) {
Preconditions.checkArgument(
file.content() == FileContent.EQUALITY_DELETES || ContentFileUtil.isDV(file),
"Must use DVs for position deletes in V%s: %s",
- formatVersion(),
+ formatVersion,
file.location());
break;
default:
- throw new IllegalArgumentException("Unsupported format version: " + formatVersion());
+ throw new IllegalArgumentException("Unsupported format version: " + formatVersion);
}
}
@@ -959,8 +963,16 @@ protected Map summary() {
return summaryBuilder.build();
}
+ // guard buffered deletes against concurrent format upgrade
+ private void validateDeleteFilesForVersion(int currentFormatVersion) {
+ for (DeleteFile file : v2Deletes) {
+ validateDeleteFileForVersion(file, currentFormatVersion);
+ }
+ }
+
@Override
public List apply(TableMetadata base, Snapshot snapshot) {
+ validateDeleteFilesForVersion(base.formatVersion());
// filter any existing manifests
List filtered =
filterManager.filterManifests(
diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
index 593dbc570b8a..2b55bcbeab22 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
@@ -223,7 +223,7 @@ public Set map(
* @param order sort order columns, will be promoted to truncate(16)
* @return metrics configuration
*/
- private static MetricsConfig from(Map props, Schema schema, SortOrder order) {
+ public static MetricsConfig from(Map props, Schema schema, SortOrder order) {
int maxInferredDefaultColumns = maxInferredColumnDefaults(props);
Map columnModes = Maps.newHashMap();
diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
index 944e833b31d7..72c57a8bebcf 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
@@ -34,9 +34,6 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import org.apache.iceberg.stats.BaseContentStats;
-import org.apache.iceberg.stats.BaseFieldStats;
-import org.apache.iceberg.stats.ContentStats;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
@@ -482,7 +479,7 @@ public void set(int pos, T value) {
}
}
- public static ContentStats fromMetrics(Schema schema, Metrics metrics) {
+ static ContentStats fromMetrics(Schema schema, Metrics metrics) {
if (null == metrics) {
return null;
}
diff --git a/core/src/main/java/org/apache/iceberg/PartitionsTable.java b/core/src/main/java/org/apache/iceberg/PartitionsTable.java
index 09c6e7893b7e..10366db5a55d 100644
--- a/core/src/main/java/org/apache/iceberg/PartitionsTable.java
+++ b/core/src/main/java/org/apache/iceberg/PartitionsTable.java
@@ -27,7 +27,9 @@
import org.apache.iceberg.expressions.ManifestEvaluator;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.ParallelIterable;
import org.apache.iceberg.util.PartitionUtil;
@@ -37,6 +39,58 @@
/** A {@link Table} implementation that exposes a table's partitions as rows. */
public class PartitionsTable extends BaseMetadataTable {
+ private static final int PARTITION_FIELD_ID = 1;
+
+ private static final Types.NestedField SPEC_ID =
+ Types.NestedField.required(4, "spec_id", Types.IntegerType.get());
+ private static final Types.NestedField RECORD_COUNT =
+ Types.NestedField.required(
+ 2, "record_count", Types.LongType.get(), "Count of records in data files");
+ private static final Types.NestedField FILE_COUNT =
+ Types.NestedField.required(3, "file_count", Types.IntegerType.get(), "Count of data files");
+ private static final Types.NestedField TOTAL_DATA_FILE_SIZE_IN_BYTES =
+ Types.NestedField.required(
+ 11,
+ "total_data_file_size_in_bytes",
+ Types.LongType.get(),
+ "Total size in bytes of data files");
+ private static final Types.NestedField POSITION_DELETE_RECORD_COUNT =
+ Types.NestedField.required(
+ 5,
+ "position_delete_record_count",
+ Types.LongType.get(),
+ "Count of records in position delete files");
+ private static final Types.NestedField POSITION_DELETE_FILE_COUNT =
+ Types.NestedField.required(
+ 6,
+ "position_delete_file_count",
+ Types.IntegerType.get(),
+ "Count of position delete files");
+ private static final Types.NestedField EQUALITY_DELETE_RECORD_COUNT =
+ Types.NestedField.required(
+ 7,
+ "equality_delete_record_count",
+ Types.LongType.get(),
+ "Count of records in equality delete files");
+ private static final Types.NestedField EQUALITY_DELETE_FILE_COUNT =
+ Types.NestedField.required(
+ 8,
+ "equality_delete_file_count",
+ Types.IntegerType.get(),
+ "Count of equality delete files");
+ private static final Types.NestedField LAST_UPDATED_AT =
+ Types.NestedField.optional(
+ 9,
+ "last_updated_at",
+ Types.TimestampType.withZone(),
+ "Commit time of snapshot that last updated this partition");
+ private static final Types.NestedField LAST_UPDATED_SNAPSHOT_ID =
+ Types.NestedField.optional(
+ 10,
+ "last_updated_snapshot_id",
+ Types.LongType.get(),
+ "Id of snapshot that last updated this partition");
+
private final Schema schema;
private final boolean unpartitionedTable;
@@ -50,47 +104,18 @@ public class PartitionsTable extends BaseMetadataTable {
this.schema =
new Schema(
- Types.NestedField.required(1, "partition", Partitioning.partitionType(table)),
- Types.NestedField.required(4, "spec_id", Types.IntegerType.get()),
- Types.NestedField.required(
- 2, "record_count", Types.LongType.get(), "Count of records in data files"),
- Types.NestedField.required(
- 3, "file_count", Types.IntegerType.get(), "Count of data files"),
- Types.NestedField.required(
- 11,
- "total_data_file_size_in_bytes",
- Types.LongType.get(),
- "Total size in bytes of data files"),
Types.NestedField.required(
- 5,
- "position_delete_record_count",
- Types.LongType.get(),
- "Count of records in position delete files"),
- Types.NestedField.required(
- 6,
- "position_delete_file_count",
- Types.IntegerType.get(),
- "Count of position delete files"),
- Types.NestedField.required(
- 7,
- "equality_delete_record_count",
- Types.LongType.get(),
- "Count of records in equality delete files"),
- Types.NestedField.required(
- 8,
- "equality_delete_file_count",
- Types.IntegerType.get(),
- "Count of equality delete files"),
- Types.NestedField.optional(
- 9,
- "last_updated_at",
- Types.TimestampType.withZone(),
- "Commit time of snapshot that last updated this partition"),
- Types.NestedField.optional(
- 10,
- "last_updated_snapshot_id",
- Types.LongType.get(),
- "Id of snapshot that last updated this partition"));
+ PARTITION_FIELD_ID, "partition", Partitioning.partitionType(table)),
+ SPEC_ID,
+ RECORD_COUNT,
+ FILE_COUNT,
+ TOTAL_DATA_FILE_SIZE_IN_BYTES,
+ POSITION_DELETE_RECORD_COUNT,
+ POSITION_DELETE_FILE_COUNT,
+ EQUALITY_DELETE_RECORD_COUNT,
+ EQUALITY_DELETE_FILE_COUNT,
+ LAST_UPDATED_AT,
+ LAST_UPDATED_SNAPSHOT_ID);
this.unpartitionedTable = Partitioning.partitionType(table).fields().isEmpty();
}
@@ -102,16 +127,18 @@ public TableScan newScan() {
@Override
public Schema schema() {
if (unpartitionedTable) {
- return schema.select(
- "record_count",
- "file_count",
- "total_data_file_size_in_bytes",
- "position_delete_record_count",
- "position_delete_file_count",
- "equality_delete_record_count",
- "equality_delete_file_count",
- "last_updated_at",
- "last_updated_snapshot_id");
+ return TypeUtil.select(
+ schema,
+ ImmutableSet.of(
+ RECORD_COUNT.fieldId(),
+ FILE_COUNT.fieldId(),
+ TOTAL_DATA_FILE_SIZE_IN_BYTES.fieldId(),
+ POSITION_DELETE_RECORD_COUNT.fieldId(),
+ POSITION_DELETE_FILE_COUNT.fieldId(),
+ EQUALITY_DELETE_RECORD_COUNT.fieldId(),
+ EQUALITY_DELETE_FILE_COUNT.fieldId(),
+ LAST_UPDATED_AT.fieldId(),
+ LAST_UPDATED_SNAPSHOT_ID.fieldId()));
}
return schema;
}
diff --git a/api/src/main/java/org/apache/iceberg/stats/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java
similarity index 98%
rename from api/src/main/java/org/apache/iceberg/stats/StatsUtil.java
rename to core/src/main/java/org/apache/iceberg/StatsUtil.java
index 2ff52f92bdda..39fef3d372d3 100644
--- a/api/src/main/java/org/apache/iceberg/stats/StatsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
import static org.apache.iceberg.types.Types.NestedField.optional;
@@ -25,7 +25,6 @@
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.types.TypeUtil;
@@ -33,7 +32,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class StatsUtil {
+class StatsUtil {
private static final Logger LOG = LoggerFactory.getLogger(StatsUtil.class);
// the number of reserved field IDs from the reserved field ID space as defined in
// https://iceberg.apache.org/spec/#reserved-field-ids
diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java
index 1f778984af17..021ef95d9122 100644
--- a/core/src/main/java/org/apache/iceberg/TableProperties.java
+++ b/core/src/main/java/org/apache/iceberg/TableProperties.java
@@ -135,6 +135,10 @@ private TableProperties() {}
"write.delete.parquet.page-size-bytes";
public static final int PARQUET_PAGE_SIZE_BYTES_DEFAULT = 1024 * 1024; // 1 MB
+ public static final String PARQUET_PAGE_VERSION = "write.parquet.page-version";
+ public static final String DELETE_PARQUET_PAGE_VERSION = "write.delete.parquet.page-version";
+ public static final String PARQUET_PAGE_VERSION_DEFAULT = "v1";
+
public static final String PARQUET_PAGE_ROW_LIMIT = "write.parquet.page-row-limit";
public static final String DELETE_PARQUET_PAGE_ROW_LIMIT = "write.delete.parquet.page-row-limit";
public static final int PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20_000;
@@ -154,6 +158,12 @@ private TableProperties() {}
"write.delete.parquet.compression-level";
public static final String PARQUET_COMPRESSION_LEVEL_DEFAULT = null;
+ public static final String PARQUET_SHRED_VARIANTS = "write.parquet.shred-variants";
+ public static final boolean PARQUET_SHRED_VARIANTS_DEFAULT = false;
+ public static final String PARQUET_VARIANT_BUFFER_SIZE =
+ "write.parquet.variant-inference-buffer-size";
+ public static final int PARQUET_VARIANT_BUFFER_SIZE_DEFAULT = 100;
+
public static final String PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT =
"write.parquet.row-group-check-min-record-count";
public static final String DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT =
@@ -346,6 +356,10 @@ private TableProperties() {}
public static final String SPARK_WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema";
public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false;
+ public static final String SPARK_WRITE_AUTO_SCHEMA_EVOLUTION =
+ "write.spark.auto-schema-evolution.enabled";
+ public static final boolean SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true;
+
public static final String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES =
"write.spark.advisory-partition-size-bytes";
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java
new file mode 100644
index 000000000000..d9ae100ac651
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.types.Types;
+
+/** A file tracked by a manifest. */
+interface TrackedFile {
+ Types.NestedField TRACKING =
+ Types.NestedField.required(
+ 147, "tracking", Tracking.schema(), "Tracking information for this entry");
+ Types.NestedField CONTENT_TYPE =
+ Types.NestedField.required(
+ 134,
+ "content_type",
+ Types.IntegerType.get(),
+ "Type of content: 0=DATA, 2=EQUALITY_DELETES, 3=DATA_MANIFEST, 4=DELETE_MANIFEST");
+ Types.NestedField LOCATION =
+ Types.NestedField.required(100, "location", Types.StringType.get(), "Location of the file");
+ Types.NestedField FILE_FORMAT =
+ Types.NestedField.required(
+ 101,
+ "file_format",
+ Types.StringType.get(),
+ "String file format name: avro, orc, or parquet");
+ Types.NestedField RECORD_COUNT =
+ Types.NestedField.required(
+ 103, "record_count", Types.LongType.get(), "Number of records in this file");
+ Types.NestedField FILE_SIZE_IN_BYTES =
+ Types.NestedField.required(
+ 104, "file_size_in_bytes", Types.LongType.get(), "Total file size in bytes");
+ Types.NestedField SPEC_ID =
+ Types.NestedField.optional(
+ 141, "spec_id", Types.IntegerType.get(), "Spec ID used to partition the file");
+
+ int CONTENT_STATS_ID = 146;
+ String CONTENT_STATS_NAME = "content_stats";
+ String CONTENT_STATS_DOC = "Content statistics for this entry";
+
+ Types.NestedField SORT_ORDER_ID =
+ Types.NestedField.optional(
+ 140, "sort_order_id", Types.IntegerType.get(), "ID of the sort order for this file");
+ Types.NestedField DELETION_VECTOR =
+ Types.NestedField.optional(
+ 148, "deletion_vector", DeletionVector.schema(), "Deletion vector for the data file");
+ Types.NestedField MANIFEST_INFO =
+ Types.NestedField.optional(
+ 150,
+ "manifest_info",
+ ManifestInfo.schema(),
+ "Metadata fields specific to manifest files");
+ Types.NestedField KEY_METADATA =
+ Types.NestedField.optional(
+ 131,
+ "key_metadata",
+ Types.BinaryType.get(),
+ "Implementation-specific key metadata for encryption");
+ Types.NestedField SPLIT_OFFSETS =
+ Types.NestedField.optional(
+ 132,
+ "split_offsets",
+ Types.ListType.ofRequired(133, Types.LongType.get()),
+ "Split offsets for the data file");
+ Types.NestedField EQUALITY_IDS =
+ Types.NestedField.optional(
+ 135,
+ "equality_ids",
+ Types.ListType.ofRequired(136, Types.IntegerType.get()),
+ "Field ids used to determine row equality in equality delete files");
+
+ static Types.StructType schemaWithContentStats(Types.StructType contentStatsType) {
+ return Types.StructType.of(
+ TRACKING,
+ CONTENT_TYPE,
+ LOCATION,
+ FILE_FORMAT,
+ RECORD_COUNT,
+ FILE_SIZE_IN_BYTES,
+ SPEC_ID,
+ Types.NestedField.optional(
+ CONTENT_STATS_ID, CONTENT_STATS_NAME, contentStatsType, CONTENT_STATS_DOC),
+ SORT_ORDER_ID,
+ DELETION_VECTOR,
+ MANIFEST_INFO,
+ KEY_METADATA,
+ SPLIT_OFFSETS,
+ EQUALITY_IDS);
+ }
+
+ /** Returns the tracking information for this entry. */
+ Tracking tracking();
+
+ /** Returns the type of content stored by this entry. */
+ FileContent contentType();
+
+ /** Returns the location of the file. */
+ String location();
+
+ /** Returns the format of the file. */
+ FileFormat fileFormat();
+
+ /** Returns the number of records in this file. */
+ long recordCount();
+
+ /** Returns the total file size in bytes. */
+ long fileSizeInBytes();
+
+ /** Returns the ID of the partition spec used to partition this file, or null. */
+ Integer specId();
+
+ /** Returns the content stats for this entry. */
+ ContentStats contentStats();
+
+ /** Returns the ID of the sort order for this file, or null. */
+ Integer sortOrderId();
+
+ /** Returns the deletion vector for this entry, or null if there is no deletion vector. */
+ DeletionVector deletionVector();
+
+ /** Returns the manifest summary information, or null for non-manifest entries. */
+ ManifestInfo manifestInfo();
+
+ /** Returns encryption key metadata, or null if the file is not encrypted. */
+ ByteBuffer keyMetadata();
+
+ /** Returns the list of recommended split locations, or null. */
+ List splitOffsets();
+
+ /** Returns the set of field IDs used for equality comparison in equality delete files. */
+ List equalityIds();
+
+ /** Copies this tracked file. */
+ TrackedFile copy();
+
+ /**
+ * Copies this tracked file with stats only for specific columns.
+ *
+ * @param requestedColumnIds table field IDs for which to keep stats
+ */
+ TrackedFile copyWithStats(Set requestedColumnIds);
+
+ /** Copies this tracked file without stats. */
+ default TrackedFile copyWithoutStats() {
+ return copyWithStats(Collections.emptySet());
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
new file mode 100644
index 000000000000..ba9fd362038a
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.avro.SupportsIndexProjection;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ArrayUtil;
+import org.apache.iceberg.util.ByteBuffers;
+
+/** Mutable {@link StructLike} implementation of {@link TrackedFile}. */
+class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, Serializable {
+ private static final Types.StructType EMPTY_STRUCT_TYPE = Types.StructType.of();
+
+ private static final Types.StructType BASE_TYPE =
+ Types.StructType.of(
+ TrackedFile.TRACKING,
+ TrackedFile.CONTENT_TYPE,
+ TrackedFile.LOCATION,
+ TrackedFile.FILE_FORMAT,
+ TrackedFile.RECORD_COUNT,
+ TrackedFile.FILE_SIZE_IN_BYTES,
+ TrackedFile.SPEC_ID,
+ Types.NestedField.optional(
+ TrackedFile.CONTENT_STATS_ID,
+ TrackedFile.CONTENT_STATS_NAME,
+ EMPTY_STRUCT_TYPE,
+ TrackedFile.CONTENT_STATS_DOC),
+ TrackedFile.SORT_ORDER_ID,
+ TrackedFile.DELETION_VECTOR,
+ TrackedFile.MANIFEST_INFO,
+ TrackedFile.KEY_METADATA,
+ TrackedFile.SPLIT_OFFSETS,
+ TrackedFile.EQUALITY_IDS);
+
+ private FileContent contentType = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private long recordCount = -1L;
+ private long fileSizeInBytes = -1L;
+ private Integer specId = null;
+
+ // optional fields
+ private Tracking tracking = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private byte[] keyMetadata = null;
+ private long[] splitOffsets = null;
+ private int[] equalityIds = null;
+
+ /** Used by internal readers to instantiate this class with a projection schema. */
+ TrackedFileStruct(Types.StructType projection) {
+ super(BASE_TYPE, projection);
+ }
+
+ /** No-projection constructor for direct construction. */
+ TrackedFileStruct() {
+ super(BASE_TYPE.fields().size());
+ }
+
+ /** Constructor that accepts required fields. */
+ TrackedFileStruct(
+ Tracking tracking,
+ FileContent contentType,
+ String location,
+ FileFormat fileFormat,
+ long recordCount,
+ long fileSizeInBytes) {
+ super(BASE_TYPE.fields().size());
+ this.tracking = tracking;
+ this.contentType = contentType;
+ this.location = location;
+ this.fileFormat = fileFormat;
+ this.recordCount = recordCount;
+ this.fileSizeInBytes = fileSizeInBytes;
+ }
+
+ /** Copy constructor. */
+ private TrackedFileStruct(TrackedFileStruct toCopy, boolean withStats, Set statsIds) {
+ super(toCopy);
+ this.contentType = toCopy.contentType;
+ this.location = toCopy.location;
+ this.fileFormat = toCopy.fileFormat;
+ this.recordCount = toCopy.recordCount;
+ this.fileSizeInBytes = toCopy.fileSizeInBytes;
+ this.specId = toCopy.specId;
+
+ this.tracking = toCopy.tracking != null ? toCopy.tracking.copy() : null;
+
+ this.sortOrderId = toCopy.sortOrderId;
+ this.deletionVector = toCopy.deletionVector != null ? toCopy.deletionVector.copy() : null;
+
+ if (withStats && toCopy.contentStats != null) {
+ ContentStats filtered = BaseContentStats.buildFrom(toCopy.contentStats, statsIds).build();
+ this.contentStats = filtered.fieldStats().isEmpty() ? null : filtered;
+ } else {
+ this.contentStats = null;
+ }
+
+ this.manifestInfo = toCopy.manifestInfo != null ? toCopy.manifestInfo.copy() : null;
+ this.keyMetadata =
+ toCopy.keyMetadata != null
+ ? Arrays.copyOf(toCopy.keyMetadata, toCopy.keyMetadata.length)
+ : null;
+ this.splitOffsets =
+ toCopy.splitOffsets != null
+ ? Arrays.copyOf(toCopy.splitOffsets, toCopy.splitOffsets.length)
+ : null;
+ this.equalityIds =
+ toCopy.equalityIds != null
+ ? Arrays.copyOf(toCopy.equalityIds, toCopy.equalityIds.length)
+ : null;
+ }
+
+ @Override
+ public Tracking tracking() {
+ return tracking;
+ }
+
+ @Override
+ public FileContent contentType() {
+ return contentType;
+ }
+
+ @Override
+ public String location() {
+ return location;
+ }
+
+ @Override
+ public FileFormat fileFormat() {
+ return fileFormat;
+ }
+
+ @Override
+ public long recordCount() {
+ return recordCount;
+ }
+
+ @Override
+ public long fileSizeInBytes() {
+ return fileSizeInBytes;
+ }
+
+ @Override
+ public Integer specId() {
+ return specId;
+ }
+
+ @Override
+ public ContentStats contentStats() {
+ return contentStats;
+ }
+
+ @Override
+ public Integer sortOrderId() {
+ return sortOrderId;
+ }
+
+ @Override
+ public DeletionVector deletionVector() {
+ return deletionVector;
+ }
+
+ @Override
+ public ManifestInfo manifestInfo() {
+ return manifestInfo;
+ }
+
+ @Override
+ public ByteBuffer keyMetadata() {
+ return keyMetadata != null ? ByteBuffer.wrap(keyMetadata) : null;
+ }
+
+ @Override
+ public List splitOffsets() {
+ return splitOffsets != null ? ArrayUtil.toUnmodifiableLongList(splitOffsets) : null;
+ }
+
+ @Override
+ public List equalityIds() {
+ return equalityIds != null ? ArrayUtil.toUnmodifiableIntList(equalityIds) : null;
+ }
+
+ @Override
+ public TrackedFile copy() {
+ return new TrackedFileStruct(this, true, null);
+ }
+
+ @Override
+ public TrackedFile copyWithStats(Set requestedColumnIds) {
+ return new TrackedFileStruct(this, true, requestedColumnIds);
+ }
+
+ @Override
+ protected T internalGet(int pos, Class javaClass) {
+ return javaClass.cast(getByPos(pos));
+ }
+
+ private Object getByPos(int pos) {
+ switch (pos) {
+ case 0:
+ return tracking;
+ case 1:
+ return contentType != null ? contentType.id() : null;
+ case 2:
+ return location;
+ case 3:
+ return fileFormat != null ? fileFormat.toString() : null;
+ case 4:
+ return recordCount;
+ case 5:
+ return fileSizeInBytes;
+ case 6:
+ return specId;
+ case 7:
+ return contentStats;
+ case 8:
+ return sortOrderId;
+ case 9:
+ return deletionVector;
+ case 10:
+ return manifestInfo;
+ case 11:
+ return keyMetadata();
+ case 12:
+ return splitOffsets();
+ case 13:
+ return equalityIds();
+ default:
+ throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
+ }
+ }
+
+ @Override
+ protected void internalSet(int pos, T value) {
+ switch (pos) {
+ case 0:
+ this.tracking = (Tracking) value;
+ break;
+ case 1:
+ this.contentType = FileContent.fromId((Integer) value);
+ break;
+ case 2:
+ // always coerce to String for Serializable
+ this.location = value.toString();
+ break;
+ case 3:
+ this.fileFormat = FileFormat.fromString(value.toString());
+ break;
+ case 4:
+ this.recordCount = (Long) value;
+ break;
+ case 5:
+ this.fileSizeInBytes = (Long) value;
+ break;
+ case 6:
+ this.specId = (Integer) value;
+ break;
+ case 7:
+ this.contentStats = (ContentStats) value;
+ break;
+ case 8:
+ this.sortOrderId = (Integer) value;
+ break;
+ case 9:
+ this.deletionVector = (DeletionVector) value;
+ break;
+ case 10:
+ this.manifestInfo = (ManifestInfo) value;
+ break;
+ case 11:
+ this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value);
+ break;
+ case 12:
+ this.splitOffsets = ArrayUtil.toLongArray((List) value);
+ break;
+ case 13:
+ this.equalityIds = ArrayUtil.toIntArray((List) value);
+ break;
+ default:
+ // ignore the object, it must be from a newer version of the format
+ }
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("content", contentType != null ? contentType.lowerCaseName() : null)
+ .add("location", location)
+ .add("file_format", fileFormat)
+ .add("record_count", recordCount)
+ .add("file_size_in_bytes", fileSizeInBytes)
+ .add("spec_id", specId())
+ .add("tracking", tracking)
+ .add("content_stats", contentStats)
+ .add("sort_order_id", sortOrderId)
+ .add("deletion_vector", deletionVector)
+ .add("manifest_info", manifestInfo)
+ .add("key_metadata", keyMetadata == null ? "null" : "(redacted)")
+ .add("split_offsets", splitOffsets == null ? "null" : splitOffsets())
+ .add("equality_ids", equalityIds == null ? "null" : equalityIds())
+ .toString();
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/Tracking.java b/core/src/main/java/org/apache/iceberg/Tracking.java
new file mode 100644
index 000000000000..8003ed82ea9c
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/Tracking.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import org.apache.iceberg.types.Types;
+
+/** Tracking information for a manifest entry. */
+interface Tracking {
+ Types.NestedField STATUS =
+ Types.NestedField.required(
+ 0,
+ "status",
+ Types.IntegerType.get(),
+ "Entry status: 0=existing, 1=added, 2=deleted, 3=replaced");
+ Types.NestedField SNAPSHOT_ID =
+ Types.NestedField.optional(
+ 1,
+ "snapshot_id",
+ Types.LongType.get(),
+ "Snapshot ID where the file was added or deleted");
+ Types.NestedField SEQUENCE_NUMBER =
+ Types.NestedField.optional(
+ 3, "sequence_number", Types.LongType.get(), "Data sequence number of the file");
+ Types.NestedField FILE_SEQUENCE_NUMBER =
+ Types.NestedField.optional(
+ 4,
+ "file_sequence_number",
+ Types.LongType.get(),
+ "File sequence number indicating when the file was added");
+ Types.NestedField DV_SNAPSHOT_ID =
+ Types.NestedField.optional(
+ 5,
+ "dv_snapshot_id",
+ Types.LongType.get(),
+ "Snapshot ID where the DV was added; null if there is no DV");
+ Types.NestedField FIRST_ROW_ID =
+ Types.NestedField.optional(
+ 142, "first_row_id", Types.LongType.get(), "ID of the first row in the data file");
+ Types.NestedField DELETED_POSITIONS =
+ Types.NestedField.optional(
+ 6,
+ "deleted_positions",
+ Types.BinaryType.get(),
+ "Bitmap of positions deleted in this snapshot");
+ Types.NestedField REPLACED_POSITIONS =
+ Types.NestedField.optional(
+ 7,
+ "replaced_positions",
+ Types.BinaryType.get(),
+ "Bitmap of positions replaced in this snapshot");
+
+ static Types.StructType schema() {
+ return Types.StructType.of(
+ STATUS,
+ SNAPSHOT_ID,
+ SEQUENCE_NUMBER,
+ FILE_SEQUENCE_NUMBER,
+ DV_SNAPSHOT_ID,
+ FIRST_ROW_ID,
+ DELETED_POSITIONS,
+ REPLACED_POSITIONS);
+ }
+
+ /** Returns the status of the entry. */
+ EntryStatus status();
+
+ /** Returns whether this entry is live. */
+ default boolean isLive() {
+ return status() == EntryStatus.ADDED || status() == EntryStatus.EXISTING;
+ }
+
+ /** Returns the snapshot ID where the file was added or deleted. */
+ Long snapshotId();
+
+ /** Returns the data sequence number of the file. */
+ Long dataSequenceNumber();
+
+ /** Returns the file sequence number indicating when the file was added. */
+ Long fileSequenceNumber();
+
+ /** Returns the snapshot ID where the DV was added; null if there is no DV. */
+ Long dvSnapshotId();
+
+ /** Returns the ID of the first row in the data file. */
+ Long firstRowId();
+
+ /** Returns the bitmap of positions deleted in this snapshot. */
+ ByteBuffer deletedPositions();
+
+ /** Returns the bitmap of positions replaced in this snapshot. */
+ ByteBuffer replacedPositions();
+
+ /** Returns the manifest location this entry was read from, or null. */
+ String manifestLocation();
+
+ /** Returns the ordinal position of this entry within the manifest. */
+ long manifestPos();
+
+ /** Copies this tracking information. */
+ Tracking copy();
+}
diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java
new file mode 100644
index 000000000000..65513c8d4a7c
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.iceberg.avro.SupportsIndexProjection;
+import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ByteBuffers;
+
+/** Mutable {@link StructLike} implementation of {@link Tracking}. */
+class TrackingStruct extends SupportsIndexProjection implements Tracking, Serializable {
+ private static final Types.StructType BASE_TYPE =
+ Types.StructType.of(
+ Tracking.STATUS,
+ Tracking.SNAPSHOT_ID,
+ Tracking.SEQUENCE_NUMBER,
+ Tracking.FILE_SEQUENCE_NUMBER,
+ Tracking.DV_SNAPSHOT_ID,
+ Tracking.FIRST_ROW_ID,
+ Tracking.DELETED_POSITIONS,
+ Tracking.REPLACED_POSITIONS,
+ MetadataColumns.ROW_POSITION);
+
+ private EntryStatus status = null;
+ private Long snapshotId = null;
+ private Long dataSequenceNumber = null;
+ private Long fileSequenceNumber = null;
+ private Long dvSnapshotId = null;
+ private Long firstRowId = null;
+ private byte[] deletedPositions = null;
+ private byte[] replacedPositions = null;
+
+ // set by manifest readers, not written to manifests
+ private String manifestLocation = null;
+ private long manifestPos = -1L;
+
+ TrackingStruct(Types.StructType type) {
+ super(BASE_TYPE, type);
+ }
+
+ private TrackingStruct(TrackingStruct toCopy) {
+ super(toCopy);
+ this.status = toCopy.status;
+ this.snapshotId = toCopy.snapshotId;
+ this.dataSequenceNumber = toCopy.dataSequenceNumber;
+ this.fileSequenceNumber = toCopy.fileSequenceNumber;
+ this.dvSnapshotId = toCopy.dvSnapshotId;
+ this.firstRowId = toCopy.firstRowId;
+ this.deletedPositions =
+ toCopy.deletedPositions != null
+ ? Arrays.copyOf(toCopy.deletedPositions, toCopy.deletedPositions.length)
+ : null;
+ this.replacedPositions =
+ toCopy.replacedPositions != null
+ ? Arrays.copyOf(toCopy.replacedPositions, toCopy.replacedPositions.length)
+ : null;
+ this.manifestLocation = toCopy.manifestLocation;
+ this.manifestPos = toCopy.manifestPos;
+ }
+
+ private TrackingStruct(
+ EntryStatus status,
+ Long snapshotId,
+ Long dataSequenceNumber,
+ Long fileSequenceNumber,
+ Long dvSnapshotId,
+ Long firstRowId,
+ byte[] deletedPositions,
+ byte[] replacedPositions) {
+ super(BASE_TYPE, BASE_TYPE);
+ this.status = status;
+ this.snapshotId = snapshotId;
+ this.dataSequenceNumber = dataSequenceNumber;
+ this.fileSequenceNumber = fileSequenceNumber;
+ this.dvSnapshotId = dvSnapshotId;
+ this.firstRowId = firstRowId;
+ this.deletedPositions = deletedPositions;
+ this.replacedPositions = replacedPositions;
+ }
+
+ void inheritFrom(Tracking manifestTracking) {
+ if (manifestTracking != null) {
+ if (snapshotId == null) {
+ this.snapshotId = manifestTracking.snapshotId();
+ }
+
+ // manifests do not distinguish between data and file sequence numbers
+ Preconditions.checkArgument(
+ Objects.equals(
+ manifestTracking.dataSequenceNumber(), manifestTracking.fileSequenceNumber()),
+ "Manifest data and file sequence numbers must be equal, got %s and %s",
+ manifestTracking.dataSequenceNumber(),
+ manifestTracking.fileSequenceNumber());
+
+ if (status == EntryStatus.ADDED) {
+ if (dataSequenceNumber == null) {
+ this.dataSequenceNumber = manifestTracking.fileSequenceNumber();
+ }
+
+ if (fileSequenceNumber == null) {
+ this.fileSequenceNumber = manifestTracking.fileSequenceNumber();
+ }
+ }
+ }
+ }
+
+ void setManifestLocation(String location) {
+ this.manifestLocation = location;
+ }
+
+ @Override
+ public EntryStatus status() {
+ return status;
+ }
+
+ @Override
+ public Long snapshotId() {
+ return snapshotId;
+ }
+
+ @Override
+ public Long dataSequenceNumber() {
+ return dataSequenceNumber;
+ }
+
+ @Override
+ public Long fileSequenceNumber() {
+ return fileSequenceNumber;
+ }
+
+ @Override
+ public Long dvSnapshotId() {
+ return dvSnapshotId;
+ }
+
+ @Override
+ public Long firstRowId() {
+ return firstRowId;
+ }
+
+ @Override
+ public ByteBuffer deletedPositions() {
+ return deletedPositions != null ? ByteBuffer.wrap(deletedPositions) : null;
+ }
+
+ @Override
+ public ByteBuffer replacedPositions() {
+ return replacedPositions != null ? ByteBuffer.wrap(replacedPositions) : null;
+ }
+
+ @Override
+ public String manifestLocation() {
+ return manifestLocation;
+ }
+
+ @Override
+ public long manifestPos() {
+ return manifestPos;
+ }
+
+ @Override
+ public TrackingStruct copy() {
+ return new TrackingStruct(this);
+ }
+
+ @Override
+ protected T internalGet(int pos, Class javaClass) {
+ return javaClass.cast(getByPos(pos));
+ }
+
+ private Object getByPos(int pos) {
+ switch (pos) {
+ case 0:
+ return status != null ? status.id() : null;
+ case 1:
+ return snapshotId();
+ case 2:
+ return dataSequenceNumber();
+ case 3:
+ return fileSequenceNumber();
+ case 4:
+ return dvSnapshotId;
+ case 5:
+ return firstRowId;
+ case 6:
+ return deletedPositions();
+ case 7:
+ return replacedPositions();
+ case 8:
+ return manifestPos;
+ default:
+ throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
+ }
+ }
+
+ @Override
+ protected void internalSet(int pos, T value) {
+ switch (pos) {
+ case 0:
+ this.status = EntryStatus.fromId((Integer) value);
+ break;
+ case 1:
+ this.snapshotId = (Long) value;
+ break;
+ case 2:
+ this.dataSequenceNumber = (Long) value;
+ break;
+ case 3:
+ this.fileSequenceNumber = (Long) value;
+ break;
+ case 4:
+ this.dvSnapshotId = (Long) value;
+ break;
+ case 5:
+ this.firstRowId = (Long) value;
+ break;
+ case 6:
+ this.deletedPositions = ByteBuffers.toByteArray((ByteBuffer) value);
+ break;
+ case 7:
+ this.replacedPositions = ByteBuffers.toByteArray((ByteBuffer) value);
+ break;
+ case 8:
+ this.manifestPos = (long) value;
+ break;
+ default:
+ // ignore the object, it must be from a newer version of the format
+ }
+ }
+
+ static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("status", status)
+ .add("snapshot_id", snapshotId == null ? "null" : snapshotId)
+ .add("data_sequence_number", dataSequenceNumber == null ? "null" : dataSequenceNumber)
+ .add("file_sequence_number", fileSequenceNumber == null ? "null" : fileSequenceNumber)
+ .add("dv_snapshot_id", dvSnapshotId == null ? "null" : dvSnapshotId)
+ .add("first_row_id", firstRowId == null ? "null" : firstRowId)
+ .add("deleted_positions", deletedPositions == null ? "null" : "(binary)")
+ .add("replaced_positions", replacedPositions == null ? "null" : "(binary)")
+ .toString();
+ }
+
+ static class Builder {
+ private EntryStatus status = null;
+ private Long snapshotId = null;
+ private Long dataSequenceNumber = null;
+ private Long fileSequenceNumber = null;
+ private Long dvSnapshotId = null;
+ private Long firstRowId = null;
+ private byte[] deletedPositions = null;
+ private byte[] replacedPositions = null;
+
+ Builder status(EntryStatus entryStatus) {
+ this.status = entryStatus;
+ return this;
+ }
+
+ Builder snapshotId(Long id) {
+ this.snapshotId = id;
+ return this;
+ }
+
+ Builder dataSequenceNumber(Long sequenceNumber) {
+ this.dataSequenceNumber = sequenceNumber;
+ return this;
+ }
+
+ Builder fileSequenceNumber(Long sequenceNumber) {
+ this.fileSequenceNumber = sequenceNumber;
+ return this;
+ }
+
+ Builder dvSnapshotId(Long id) {
+ this.dvSnapshotId = id;
+ return this;
+ }
+
+ Builder firstRowId(Long rowId) {
+ this.firstRowId = rowId;
+ return this;
+ }
+
+ Builder deletedPositions(ByteBuffer positions) {
+ this.deletedPositions = positions != null ? ByteBuffers.toByteArray(positions) : null;
+ return this;
+ }
+
+ Builder deletedPositions(byte[] positions) {
+ this.deletedPositions = positions;
+ return this;
+ }
+
+ Builder replacedPositions(ByteBuffer positions) {
+ this.replacedPositions = positions != null ? ByteBuffers.toByteArray(positions) : null;
+ return this;
+ }
+
+ Builder replacedPositions(byte[] positions) {
+ this.replacedPositions = positions;
+ return this;
+ }
+
+ TrackingStruct build() {
+ Preconditions.checkArgument(status != null, "Invalid status: null");
+ return new TrackingStruct(
+ status,
+ snapshotId,
+ dataSequenceNumber,
+ fileSequenceNumber,
+ dvSnapshotId,
+ firstRowId,
+ deletedPositions,
+ replacedPositions);
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/V2Metadata.java b/core/src/main/java/org/apache/iceberg/V2Metadata.java
index 832e5c383fe5..803905f6b42e 100644
--- a/core/src/main/java/org/apache/iceberg/V2Metadata.java
+++ b/core/src/main/java/org/apache/iceberg/V2Metadata.java
@@ -93,6 +93,7 @@ private Object get(int pos) {
case 2:
return wrapped.partitionSpecId();
case 3:
+ checkContentType(wrapped.content());
return wrapped.content().id();
case 4:
if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) {
@@ -428,6 +429,7 @@ public T get(int pos, Class javaClass) {
private Object get(int pos) {
switch (pos) {
case 0:
+ checkContentType(wrapped.content());
return wrapped.content().id();
case 1:
return wrapped.location();
@@ -589,4 +591,20 @@ public F copyWithoutStats() {
throw new UnsupportedOperationException("Cannot copy IndexedDataFile wrapper");
}
}
+
+ private static void checkContentType(ManifestContent content) {
+ Preconditions.checkArgument(
+ content == ManifestContent.DATA || content == ManifestContent.DELETES,
+ "Unsupported manifest content type for v2: %s",
+ content);
+ }
+
+ private static void checkContentType(FileContent content) {
+ Preconditions.checkArgument(
+ content == FileContent.DATA
+ || content == FileContent.POSITION_DELETES
+ || content == FileContent.EQUALITY_DELETES,
+ "Unsupported file content type for v2: %s",
+ content);
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/V3Metadata.java b/core/src/main/java/org/apache/iceberg/V3Metadata.java
index 8529d68501d4..4e67d9977e64 100644
--- a/core/src/main/java/org/apache/iceberg/V3Metadata.java
+++ b/core/src/main/java/org/apache/iceberg/V3Metadata.java
@@ -94,6 +94,7 @@ private Object get(int pos) {
case 2:
return wrapped.partitionSpecId();
case 3:
+ checkContentType(wrapped.content());
return wrapped.content().id();
case 4:
if (wrapped.sequenceNumber() == ManifestWriter.UNASSIGNED_SEQ) {
@@ -454,6 +455,7 @@ public T get(int pos, Class javaClass) {
private Object get(int pos) {
switch (pos) {
case 0:
+ checkContentType(wrapped.content());
return wrapped.content().id();
case 1:
return wrapped.location();
@@ -523,4 +525,20 @@ public Long pos() {
return null;
}
}
+
+ private static void checkContentType(ManifestContent content) {
+ Preconditions.checkArgument(
+ content == ManifestContent.DATA || content == ManifestContent.DELETES,
+ "Unsupported manifest content type for v3: %s",
+ content);
+ }
+
+ private static void checkContentType(FileContent content) {
+ Preconditions.checkArgument(
+ content == FileContent.DATA
+ || content == FileContent.POSITION_DELETES
+ || content == FileContent.EQUALITY_DELETES,
+ "Unsupported file content type for v3: %s",
+ content);
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/deletes/RoaringPositionBitmap.java b/core/src/main/java/org/apache/iceberg/deletes/RoaringPositionBitmap.java
index 3f7613c6ea58..037a38b114a6 100644
--- a/core/src/main/java/org/apache/iceberg/deletes/RoaringPositionBitmap.java
+++ b/core/src/main/java/org/apache/iceberg/deletes/RoaringPositionBitmap.java
@@ -79,14 +79,45 @@ public void set(long pos) {
}
/**
- * Sets a range of positions in the bitmap.
+ * Sets a range of positions in the bitmap. If {@code posStartInclusive} equals {@code
+ * posEndExclusive}, this method does nothing.
*
* @param posStartInclusive the start position of the range (inclusive)
* @param posEndExclusive the end position of the range (exclusive)
+ * @throws IllegalArgumentException if posStartInclusive > posEndExclusive
*/
public void setRange(long posStartInclusive, long posEndExclusive) {
- for (long pos = posStartInclusive; pos < posEndExclusive; pos++) {
- set(pos);
+ Preconditions.checkArgument(
+ posStartInclusive <= posEndExclusive,
+ "Start position must not exceed end position: [%s, %s)",
+ posStartInclusive,
+ posEndExclusive);
+
+ if (posStartInclusive == posEndExclusive) {
+ return;
+ }
+
+ validatePosition(posStartInclusive);
+ validatePosition(posEndExclusive - 1);
+
+ int startKey = key(posStartInclusive);
+ int endKey = key(posEndExclusive - 1);
+ allocateBitmapsIfNeeded(endKey + 1);
+
+ if (startKey == endKey) {
+ long lowStart = Integer.toUnsignedLong(pos32Bits(posStartInclusive));
+ long lowEnd = Integer.toUnsignedLong(pos32Bits(posEndExclusive - 1)) + 1;
+ bitmaps[startKey].add(lowStart, lowEnd);
+ } else {
+ long firstLowStart = Integer.toUnsignedLong(pos32Bits(posStartInclusive));
+ bitmaps[startKey].add(firstLowStart, 1L << 32);
+
+ for (int key = startKey + 1; key < endKey; key++) {
+ bitmaps[key].add(0L, 1L << 32);
+ }
+
+ long lastLowEnd = Integer.toUnsignedLong(pos32Bits(posEndExclusive - 1)) + 1;
+ bitmaps[endKey].add(0L, lastLowEnd);
}
}
diff --git a/core/src/main/java/org/apache/iceberg/formats/FormatModelRegistry.java b/core/src/main/java/org/apache/iceberg/formats/FormatModelRegistry.java
index 4a6b5a6cf40f..e1e93aa1fd07 100644
--- a/core/src/main/java/org/apache/iceberg/formats/FormatModelRegistry.java
+++ b/core/src/main/java/org/apache/iceberg/formats/FormatModelRegistry.java
@@ -170,6 +170,11 @@ public static FileWriterBuilder, S> equalityDelet
* records that identify rows to be deleted by file path and position, producing a {@link
* DeleteFile} that can be used for table operations.
*
+ * Note: This method is only applicable to format-version 2 tables. Format-version 3
+ * tables use deletion vectors, which are always written in Puffin format. Registered {@link
+ * FormatModel} implementations for {@link PositionDelete} are not consulted for format-version 3+
+ * tables.
+ *
* @param format the file format used for writing
* @param outputFile destination for the written data
* @return a configured delete write builder for creating a {@link PositionDeleteWriter}
diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java
index 975b5a39dfe3..2234d418de40 100644
--- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java
@@ -48,6 +48,8 @@
import org.apache.iceberg.relocated.com.google.common.base.Objects;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.view.BaseMetastoreViewCatalog;
import org.apache.iceberg.view.BaseViewOperations;
import org.apache.iceberg.view.ViewMetadata;
@@ -71,6 +73,7 @@ public class InMemoryCatalog extends BaseMetastoreViewCatalog
private String catalogName;
private String warehouseLocation;
private CloseableGroup closeableGroup;
+ private boolean uniqueTableLocation;
private Map catalogProperties;
public InMemoryCatalog() {
@@ -88,6 +91,11 @@ public String name() {
public void initialize(String name, Map properties) {
this.catalogName = name != null ? name : InMemoryCatalog.class.getSimpleName();
this.catalogProperties = ImmutableMap.copyOf(properties);
+ this.uniqueTableLocation =
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT);
String warehouse = properties.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, "");
this.warehouseLocation = warehouse.replaceAll("/*$", "");
@@ -104,8 +112,8 @@ protected TableOperations newTableOps(TableIdentifier tableIdentifier) {
@Override
protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
- return SLASH.join(
- defaultNamespaceLocation(tableIdentifier.namespace()), tableIdentifier.name());
+ String tableLocation = LocationUtil.tableLocation(tableIdentifier, uniqueTableLocation);
+ return SLASH.join(defaultNamespaceLocation(tableIdentifier.namespace()), tableLocation);
}
private String defaultNamespaceLocation(Namespace namespace) {
@@ -211,6 +219,13 @@ public boolean dropNamespace(Namespace namespace) throws NamespaceNotEmptyExcept
return false;
}
+ List childNamespaces = listNamespaces(namespace);
+ if (!childNamespaces.isEmpty()) {
+ throw new NamespaceNotEmptyException(
+ "Namespace %s is not empty. Contains %d child namespace(s).",
+ namespace, childNamespaces.size());
+ }
+
List tableIdentifiers = listTables(namespace);
if (!tableIdentifiers.isEmpty()) {
throw new NamespaceNotEmptyException(
diff --git a/core/src/main/java/org/apache/iceberg/io/BufferedFileAppender.java b/core/src/main/java/org/apache/iceberg/io/BufferedFileAppender.java
new file mode 100644
index 000000000000..8f8ef8f33b76
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/io/BufferedFileAppender.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.io;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+import org.apache.iceberg.Metrics;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+
+/**
+ * A FileAppender that buffers the first N rows, then creates a delegate appender via a factory.
+ *
+ * The factory receives the buffered rows and is responsible for creating the real appender. Row
+ * replay is handled internally. All subsequent {@link #add} calls delegate directly to the real
+ * appender.
+ *
+ *
If fewer than {@code bufferSize} rows are written before close, the factory is called with
+ * whatever rows were buffered. If no rows were written, the factory is not called and no file is
+ * created on disk. In this case, {@link #metrics()} returns {@code new Metrics(0L)} and {@link
+ * #length()} returns {@code 0L}.
+ *
+ * @param the row type
+ */
+public class BufferedFileAppender implements FileAppender {
+ private final int bufferRowCount;
+ private final Function, FileAppender> appenderFactory;
+ private final UnaryOperator copyFunc;
+ private List buffer;
+ private FileAppender delegate;
+ private boolean closed = false;
+
+ /**
+ * @param bufferRowCount number of rows to buffer before creating the delegate appender
+ * @param appenderFactory given the buffered rows, creates the delegate appender
+ */
+ public BufferedFileAppender(
+ int bufferRowCount, Function, FileAppender> appenderFactory) {
+ this(bufferRowCount, appenderFactory, UnaryOperator.identity());
+ }
+
+ /**
+ * @param bufferRowCount number of rows to buffer before creating the delegate appender
+ * @param appenderFactory given the buffered rows, creates the delegate appender
+ * @param copyFunc copies a row before buffering (needed when row objects are reused, e.g. Spark
+ * InternalRow)
+ */
+ public BufferedFileAppender(
+ int bufferRowCount,
+ Function, FileAppender> appenderFactory,
+ UnaryOperator copyFunc) {
+ Preconditions.checkArgument(
+ bufferRowCount > 0, "bufferRowCount must be > 0, got %s", bufferRowCount);
+ Preconditions.checkNotNull(appenderFactory, "appenderFactory must not be null");
+ Preconditions.checkNotNull(copyFunc, "copyFunc must not be null");
+ this.bufferRowCount = bufferRowCount;
+ this.appenderFactory = appenderFactory;
+ this.copyFunc = copyFunc;
+ this.buffer = Lists.newArrayListWithCapacity(bufferRowCount);
+ }
+
+ @Override
+ public void add(D datum) {
+ Preconditions.checkState(!closed, "Cannot add to a closed appender");
+ if (delegate != null) {
+ delegate.add(datum);
+ } else {
+ buffer.add(copyFunc.apply(datum));
+ if (buffer.size() >= bufferRowCount) {
+ initialize();
+ }
+ }
+ }
+
+ @Override
+ public Metrics metrics() {
+ Preconditions.checkState(closed, "Cannot return metrics for unclosed appender");
+ if (delegate == null) {
+ return new Metrics(0L);
+ }
+
+ return delegate.metrics();
+ }
+
+ @Override
+ public long length() {
+ if (delegate != null) {
+ return delegate.length();
+ }
+
+ // No bytes written to disk yet; data is buffered in memory
+ return 0L;
+ }
+
+ @Override
+ public List splitOffsets() {
+ if (delegate != null) {
+ return delegate.splitOffsets();
+ }
+
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ try {
+ if (delegate == null && buffer != null && !buffer.isEmpty()) {
+ initialize();
+ }
+
+ if (delegate != null) {
+ delegate.close();
+ }
+ } finally {
+ this.closed = true;
+ this.buffer = null;
+ }
+ }
+ }
+
+ private void initialize() {
+ delegate = appenderFactory.apply(buffer);
+ Preconditions.checkState(delegate != null, "appenderFactory must not return null");
+ try {
+ buffer.forEach(delegate::add);
+ } finally {
+ buffer = null;
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
index 55c00319a0cc..2d24e5598ac7 100644
--- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
@@ -88,6 +88,7 @@ public class JdbcCatalog extends BaseMetastoreViewCatalog
private Object conf;
private JdbcClientPool connections;
private Map catalogProperties;
+ private boolean uniqueTableLocation;
private final Function, FileIO> ioBuilder;
private final Function, JdbcClientPool> clientPoolBuilder;
private boolean initializeCatalogTables;
@@ -120,6 +121,11 @@ public void initialize(String name, Map properties) {
this.warehouseLocation = LocationUtil.stripTrailingSlash(inputWarehouseLocation);
this.catalogProperties = ImmutableMap.copyOf(properties);
+ this.uniqueTableLocation =
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT);
if (name != null) {
this.catalogName = name;
@@ -287,7 +293,8 @@ protected ViewOperations newViewOps(TableIdentifier viewIdentifier) {
@Override
protected String defaultWarehouseLocation(TableIdentifier table) {
- return SLASH.join(defaultNamespaceLocation(table.namespace()), table.name());
+ String tableLocation = LocationUtil.tableLocation(table, uniqueTableLocation);
+ return SLASH.join(defaultNamespaceLocation(table.namespace()), tableLocation);
}
@Override
@@ -536,6 +543,13 @@ public boolean dropNamespace(Namespace namespace) throws NamespaceNotEmptyExcept
return false;
}
+ List childNamespaces = listNamespaces(namespace);
+ if (childNamespaces != null && !childNamespaces.isEmpty()) {
+ throw new NamespaceNotEmptyException(
+ "Namespace %s is not empty. Contains %d child namespace(s).",
+ namespace, childNamespaces.size());
+ }
+
List tableIdentifiers = listTables(namespace);
if (tableIdentifiers != null && !tableIdentifiers.isEmpty()) {
throw new NamespaceNotEmptyException(
diff --git a/core/src/main/java/org/apache/iceberg/rest/Endpoint.java b/core/src/main/java/org/apache/iceberg/rest/Endpoint.java
index c2369a0fa57d..d56a14d18954 100644
--- a/core/src/main/java/org/apache/iceberg/rest/Endpoint.java
+++ b/core/src/main/java/org/apache/iceberg/rest/Endpoint.java
@@ -66,6 +66,8 @@ public class Endpoint {
Endpoint.create("POST", ResourcePaths.V1_TABLE_METRICS);
public static final Endpoint V1_TABLE_CREDENTIALS =
Endpoint.create("GET", ResourcePaths.V1_TABLE_CREDENTIALS);
+ public static final Endpoint V1_TABLE_REMOTE_SIGN =
+ Endpoint.create("POST", ResourcePaths.V1_TABLE_REMOTE_SIGN);
// table scan plan endpoints
public static final Endpoint V1_SUBMIT_TABLE_SCAN_PLAN =
diff --git a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java
index 791eb732bb7c..334bfde8abfc 100644
--- a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java
+++ b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java
@@ -30,6 +30,7 @@
import org.apache.iceberg.exceptions.NoSuchPlanTaskException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.exceptions.NoSuchViewException;
+import org.apache.iceberg.exceptions.NoSuchWarehouseException;
import org.apache.iceberg.exceptions.NotAuthorizedException;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.exceptions.RESTException;
@@ -92,6 +93,10 @@ public static Consumer defaultErrorHandler() {
return DefaultErrorHandler.INSTANCE;
}
+ public static Consumer configErrorHandler() {
+ return ConfigErrorHandler.INSTANCE;
+ }
+
public static Consumer oauthErrorHandler() {
return OAuthErrorHandler.INSTANCE;
}
@@ -295,6 +300,20 @@ public void accept(ErrorResponse error) {
}
}
+ /** Request error handler for config endpoint. */
+ private static class ConfigErrorHandler extends DefaultErrorHandler {
+ private static final ErrorHandler INSTANCE = new ConfigErrorHandler();
+
+ @Override
+ public void accept(ErrorResponse error) {
+ if (error.code() == 404 && error.type() != null) {
+ throw new NoSuchWarehouseException("%s", error.message());
+ }
+
+ super.accept(error);
+ }
+ }
+
/**
* Request error handler that handles the common cases that are included with all responses, such
* as 400, 500, etc.
diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
index 86eceba21c95..46d9177b9571 100644
--- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
+++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java
@@ -18,7 +18,6 @@
*/
package org.apache.iceberg.rest;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import java.io.IOException;
@@ -29,6 +28,7 @@
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import javax.net.ssl.HostnameVerifier;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.CredentialsProvider;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
@@ -43,6 +43,7 @@
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy;
+import org.apache.hc.client5.http.ssl.HostnameVerificationPolicy;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpHeaders;
@@ -340,32 +341,23 @@ protected T execute(
return null;
}
- String responseBody = extractResponseBodyAsString(response);
-
if (!isSuccessful(response)) {
// The provided error handler is expected to throw, but a RESTException is thrown if not.
+ String responseBody = extractResponseBodyAsString(response);
throwFailure(response, responseBody, errorHandler);
}
- if (responseBody == null) {
+ if (response.getEntity() == null) {
throw new RESTException(
"Invalid (null) response body for request (expected %s): method=%s, path=%s, status=%d",
responseType.getSimpleName(), req.method(), req.path(), response.getCode());
}
- try {
- ObjectReader reader = objectReaderCache.computeIfAbsent(responseType, mapper::readerFor);
- if (parserContext != null && !parserContext.isEmpty()) {
- reader = reader.with(parserContext.toInjectableValues());
- }
- return reader.readValue(responseBody);
- } catch (JsonProcessingException e) {
- throw new RESTException(
- e,
- "Received a success response code of %d, but failed to parse response body into %s",
- response.getCode(),
- responseType.getSimpleName());
+ ObjectReader reader = objectReaderCache.computeIfAbsent(responseType, mapper::readerFor);
+ if (parserContext != null && !parserContext.isEmpty()) {
+ reader = reader.with(parserContext.toInjectableValues());
}
+ return reader.readValue(response.getEntity().getContent());
} catch (IOException e) {
throw new RESTException(e, "Error occurred while processing %s request", req.method());
}
@@ -410,13 +402,19 @@ static HttpClientConnectionManager configureConnectionManager(Map())
.addDeserializer(LoadCredentialsResponse.class, new LoadCredentialsResponseDeserializer<>())
.addDeserializer(
- ImmutableLoadCredentialsResponse.class, new LoadCredentialsResponseDeserializer<>());
+ ImmutableLoadCredentialsResponse.class, new LoadCredentialsResponseDeserializer<>())
+ .addSerializer(RemoteSignRequest.class, new RemoteSignRequestSerializer<>())
+ .addSerializer(ImmutableRemoteSignRequest.class, new RemoteSignRequestSerializer<>())
+ .addDeserializer(RemoteSignRequest.class, new RemoteSignRequestDeserializer<>())
+ .addDeserializer(ImmutableRemoteSignRequest.class, new RemoteSignRequestDeserializer<>())
+ .addSerializer(RemoteSignResponse.class, new RemoteSignResponseSerializer<>())
+ .addSerializer(ImmutableRemoteSignResponse.class, new RemoteSignResponseSerializer<>())
+ .addDeserializer(RemoteSignResponse.class, new RemoteSignResponseDeserializer<>())
+ .addDeserializer(ImmutableRemoteSignResponse.class, new RemoteSignResponseDeserializer<>());
mapper.registerModule(module);
}
@@ -650,4 +664,39 @@ boolean isCaseSensitive() {
return caseSensitive;
}
}
+
+ static class RemoteSignRequestSerializer extends JsonSerializer {
+ @Override
+ public void serialize(T request, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException {
+ RemoteSignRequestParser.toJson(request, gen);
+ }
+ }
+
+ static class RemoteSignRequestDeserializer
+ extends JsonDeserializer {
+ @Override
+ public T deserialize(JsonParser p, DeserializationContext context) throws IOException {
+ JsonNode jsonNode = p.getCodec().readTree(p);
+ return (T) RemoteSignRequestParser.fromJson(jsonNode);
+ }
+ }
+
+ static class RemoteSignResponseSerializer
+ extends JsonSerializer {
+ @Override
+ public void serialize(T response, JsonGenerator gen, SerializerProvider serializers)
+ throws IOException {
+ RemoteSignResponseParser.toJson(response, gen);
+ }
+ }
+
+ static class RemoteSignResponseDeserializer
+ extends JsonDeserializer {
+ @Override
+ public T deserialize(JsonParser p, DeserializationContext context) throws IOException {
+ JsonNode jsonNode = p.getCodec().readTree(p);
+ return (T) RemoteSignResponseParser.fromJson(jsonNode);
+ }
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
index cbdf17a8ebbe..ec30d9de897e 100644
--- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
@@ -265,7 +265,7 @@ public void initialize(String name, Map unresolved) {
PropertyUtil.propertyAsString(
mergedProps,
RESTCatalogProperties.SNAPSHOT_LOADING_MODE,
- RESTCatalogProperties.SNAPSHOT_LOADING_MODE_DEFAULT)
+ RESTCatalogProperties.SNAPSHOT_LOADING_MODE_DEFAULT.name())
.toUpperCase(Locale.US));
this.reporter = CatalogUtil.loadMetricsReporter(mergedProps);
@@ -279,7 +279,7 @@ public void initialize(String name, Map unresolved) {
PropertyUtil.propertyAsString(
mergedProps,
RESTCatalogProperties.NAMESPACE_SEPARATOR,
- RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8);
+ RESTCatalogProperties.NAMESPACE_SEPARATOR_DEFAULT);
this.tableCache = createTableCache(mergedProps);
this.closeables.addCloseable(this.tableCache);
@@ -615,7 +615,7 @@ private RESTTable restTableForScanPlanning(
RESTCatalogProperties.ScanPlanningMode effectiveMode =
effectiveModeConfig != null
? RESTCatalogProperties.ScanPlanningMode.fromString(effectiveModeConfig)
- : RESTCatalogProperties.ScanPlanningMode.CLIENT;
+ : RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT;
if (effectiveMode == RESTCatalogProperties.ScanPlanningMode.SERVER) {
Preconditions.checkState(
@@ -1338,7 +1338,7 @@ private static ConfigResponse fetchConfig(
queryParams.build(),
ConfigResponse.class,
RESTUtil.configHeaders(properties),
- ErrorHandlers.defaultErrorHandler());
+ ErrorHandlers.configErrorHandler());
configResponse.validate();
return configResponse;
}
diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java
index 74fe9ebd7d4e..9fa273ca169f 100644
--- a/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java
+++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableScan.java
@@ -22,6 +22,7 @@
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.RemovalListener;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
@@ -47,9 +48,11 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.rest.credentials.Credential;
import org.apache.iceberg.rest.requests.PlanTableScanRequest;
+import org.apache.iceberg.rest.responses.ErrorResponse;
import org.apache.iceberg.rest.responses.FetchPlanningResultResponse;
import org.apache.iceberg.rest.responses.PlanTableScanResponse;
import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +62,6 @@ class RESTTableScan extends DataTableScan {
private static final long MIN_SLEEP_MS = 1000; // Initial delay
private static final long MAX_SLEEP_MS = 60 * 1000; // Max backoff delay (1 minute)
private static final int MAX_RETRIES = 10; // Max number of poll retries
- private static final long MAX_WAIT_TIME_MS = 5 * 60 * 1000; // Total maximum duration (5 minutes)
private static final double SCALE_FACTOR = 2.0; // Exponential scale factor
private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.io.ResolvingFileIO";
private static final Cache FILEIO_TRACKER =
@@ -217,11 +219,7 @@ private CloseableIterable planTableScan(PlanTableScanRequest planT
Endpoint.check(supportedEndpoints, Endpoint.V1_FETCH_TABLE_SCAN_PLAN);
return fetchPlanningResult();
case FAILED:
- throw new IllegalStateException(
- String.format("Received status: %s for planId: %s", PlanStatus.FAILED, planId));
- case CANCELLED:
- throw new IllegalStateException(
- String.format("Received status: %s for planId: %s", PlanStatus.CANCELLED, planId));
+ throw new IllegalStateException(failureMessage(planId, response.errorResponse()));
default:
throw new IllegalStateException(
String.format("Invalid planStatus: %s for planId: %s", planStatus, planId));
@@ -249,38 +247,72 @@ private FileIO scanFileIO(List storageCredentials) {
}
private CloseableIterable fetchPlanningResult() {
- AtomicReference result = new AtomicReference<>();
- Tasks.foreach(planId)
- .exponentialBackoff(MIN_SLEEP_MS, MAX_SLEEP_MS, MAX_WAIT_TIME_MS, SCALE_FACTOR)
- .retry(MAX_RETRIES)
- .onlyRetryOn(NotCompleteException.class)
- .onFailure(
- (id, err) -> {
- LOG.warn("Planning failed for plan ID: {}", id, err);
- cleanupPlanResources();
- })
- .throwFailureWhenFinished()
- .run(
- id -> {
- FetchPlanningResultResponse response =
- client.get(
- resourcePaths.plan(tableIdentifier, id),
- headers,
- FetchPlanningResultResponse.class,
- headers,
- ErrorHandlers.planErrorHandler(),
- parserContext);
+ long maxWaitTimeMs =
+ PropertyUtil.propertyAsLong(
+ catalogProperties,
+ RESTCatalogProperties.REST_SCAN_PLANNING_POLL_TIMEOUT_MS,
+ RESTCatalogProperties.REST_SCAN_PLANNING_POLL_TIMEOUT_MS_DEFAULT);
+ Preconditions.checkArgument(
+ maxWaitTimeMs > 0,
+ "Invalid value for %s: %s (must be positive)",
+ RESTCatalogProperties.REST_SCAN_PLANNING_POLL_TIMEOUT_MS,
+ maxWaitTimeMs);
- if (response.planStatus() == PlanStatus.SUBMITTED) {
- throw new NotCompleteException();
- } else if (response.planStatus() != PlanStatus.COMPLETED) {
- throw new IllegalStateException(
- String.format(
- "Invalid planStatus: %s for planId: %s", response.planStatus(), id));
- }
+ AtomicReference result = new AtomicReference<>();
+ try {
+ Tasks.foreach(planId)
+ .exponentialBackoff(MIN_SLEEP_MS, MAX_SLEEP_MS, maxWaitTimeMs, SCALE_FACTOR)
+ .retry(MAX_RETRIES)
+ .onlyRetryOn(NotCompleteException.class)
+ .onFailure(
+ (id, err) -> {
+ LOG.warn("Planning failed for plan ID: {}", id, err);
+ cleanupPlanResources();
+ })
+ .throwFailureWhenFinished()
+ .run(
+ id -> {
+ FetchPlanningResultResponse response =
+ client.get(
+ resourcePaths.plan(tableIdentifier, id),
+ headers,
+ FetchPlanningResultResponse.class,
+ headers,
+ ErrorHandlers.planErrorHandler(),
+ parserContext);
- result.set(response);
- });
+ switch (response.planStatus()) {
+ case COMPLETED:
+ result.set(response);
+ break;
+ case SUBMITTED:
+ throw new NotCompleteException();
+ case FAILED:
+ throw new IllegalStateException(failureMessage(id, response.errorResponse()));
+ case CANCELLED:
+ throw new IllegalStateException(
+ String.format(
+ Locale.ROOT, "Remote scan planning cancelled for planId: %s", id));
+ default:
+ throw new IllegalStateException(
+ String.format(
+ Locale.ROOT,
+ "Invalid planStatus: %s for planId: %s",
+ response.planStatus(),
+ id));
+ }
+ });
+ } catch (NotCompleteException e) {
+ throw new RemotePlanTimeoutException(
+ String.format(
+ Locale.ROOT,
+ "Remote scan planning for planId: %s did not complete within configured limits"
+ + " (timeout=%d ms, maxRetries=%d)",
+ planId,
+ maxWaitTimeMs,
+ MAX_RETRIES),
+ e);
+ }
FetchPlanningResultResponse response = result.get();
@@ -290,6 +322,21 @@ private CloseableIterable fetchPlanningResult() {
return scanTasksIterable(response.planTasks(), response.fileScanTasks());
}
+ private static String failureMessage(String planId, ErrorResponse error) {
+ // If a FAILED response lacks the expected error payload, still return a useful error
+ // message instead of throwing.
+ String type = error != null ? error.type() : "unknown";
+ int code = error != null ? error.code() : 0;
+ String message = error != null ? error.message() : "unknown";
+ return String.format(
+ Locale.ROOT,
+ "Remote scan planning failed for planId: %s: %s (code=%d): %s",
+ planId,
+ type,
+ code,
+ message);
+ }
+
private CloseableIterable scanTasksIterable(
List planTasks, List fileScanTasks) {
if (planTasks != null && !planTasks.isEmpty()) {
diff --git a/core/src/main/java/org/apache/iceberg/rest/RemotePlanTimeoutException.java b/core/src/main/java/org/apache/iceberg/rest/RemotePlanTimeoutException.java
new file mode 100644
index 000000000000..e0f01aadd612
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/rest/RemotePlanTimeoutException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest;
+
+/** Thrown when server-side scan planning does not complete before the client deadline. */
+class RemotePlanTimeoutException extends RuntimeException {
+ RemotePlanTimeoutException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java
index 0fc55c1a44d8..be2fde22053d 100644
--- a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java
+++ b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java
@@ -35,6 +35,8 @@ public class ResourcePaths {
public static final String V1_TABLE = "/v1/{prefix}/namespaces/{namespace}/tables/{table}";
public static final String V1_TABLE_CREDENTIALS =
"/v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials";
+ public static final String V1_TABLE_REMOTE_SIGN =
+ "/v1/{prefix}/namespaces/{namespace}/tables/{table}/sign";
public static final String V1_TABLE_REGISTER = "/v1/{prefix}/namespaces/{namespace}/register";
public static final String V1_TABLE_METRICS =
"/v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics";
@@ -57,7 +59,7 @@ public static ResourcePaths forCatalogProperties(Map properties)
PropertyUtil.propertyAsString(
properties,
RESTCatalogProperties.NAMESPACE_SEPARATOR,
- RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8));
+ RESTCatalogProperties.NAMESPACE_SEPARATOR_DEFAULT));
}
public static String config() {
@@ -130,6 +132,17 @@ public String metrics(TableIdentifier identifier) {
"metrics");
}
+ public String remoteSign(TableIdentifier identifier) {
+ return SLASH.join(
+ "v1",
+ prefix,
+ "namespaces",
+ pathEncode(identifier.namespace()),
+ "tables",
+ RESTUtil.encodeString(identifier.name()),
+ "sign");
+ }
+
public String commitTransaction() {
return SLASH.join("v1", prefix, "transactions", "commit");
}
diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java
index c2b47e6e944f..7a244bff70f6 100644
--- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java
+++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java
@@ -529,6 +529,7 @@ public Pair refresh(RESTClient client) {
.from(config())
.token(response.token())
.tokenType(response.issuedTokenType())
+ .expiresAtMillis(OAuth2Util.expiresAtMillis(response.token()))
.build();
Map currentHeaders = this.headers;
this.headers = RESTUtil.merge(currentHeaders, authHeaders(config.token()));
@@ -618,6 +619,7 @@ public static AuthSession fromAccessToken(
.from(parent.config())
.token(token)
.tokenType(OAuth2Properties.ACCESS_TOKEN_TYPE)
+ .expiresAtMillis(OAuth2Util.expiresAtMillis(token))
.build());
long startTimeMillis = System.currentTimeMillis();
@@ -699,6 +701,7 @@ private static AuthSession fromTokenResponse(
.token(response.token())
.tokenType(issuedTokenType)
.credential(credential)
+ .expiresAtMillis(OAuth2Util.expiresAtMillis(response.token()))
.build());
Long expiresAtMillis = session.expiresAtMillis();
diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/TLSConfigurer.java b/core/src/main/java/org/apache/iceberg/rest/auth/TLSConfigurer.java
index da22b5cb4c6a..4b3ed3a74c24 100644
--- a/core/src/main/java/org/apache/iceberg/rest/auth/TLSConfigurer.java
+++ b/core/src/main/java/org/apache/iceberg/rest/auth/TLSConfigurer.java
@@ -19,9 +19,9 @@
package org.apache.iceberg.rest.auth;
import java.util.Map;
+import javax.annotation.Nullable;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
-import org.apache.hc.client5.http.ssl.HttpsSupport;
import org.apache.hc.core5.ssl.SSLContexts;
public interface TLSConfigurer {
@@ -32,8 +32,16 @@ default SSLContext sslContext() {
return SSLContexts.createDefault();
}
+ /**
+ * Returns a custom {@link HostnameVerifier} to use for hostname verification, or {@code null} to
+ * use the default JSSE built-in hostname verifier.
+ *
+ * If a non-null verifier is returned, only the custom verifier is executed and the JSSE
+ * built-in hostname verifier won't be executed.
+ */
+ @Nullable
default HostnameVerifier hostnameVerifier() {
- return HttpsSupport.getDefaultHostnameVerifier();
+ return null;
}
default String[] supportedProtocols() {
diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequest.java b/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequest.java
new file mode 100644
index 000000000000..561007c480eb
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest.requests;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.iceberg.rest.RESTRequest;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public interface RemoteSignRequest extends RESTRequest {
+ String region();
+
+ String method();
+
+ URI uri();
+
+ Map> headers();
+
+ Map properties();
+
+ @Value.Default
+ @Nullable
+ default String body() {
+ return null;
+ }
+
+ @Nullable
+ String provider();
+
+ @Override
+ default void validate() {}
+}
diff --git a/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequestParser.java b/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequestParser.java
new file mode 100644
index 000000000000..61b44cc177d1
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/rest/requests/RemoteSignRequestParser.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest.requests;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.JsonUtil;
+
+public class RemoteSignRequestParser {
+
+ private static final String REGION = "region";
+ private static final String METHOD = "method";
+ private static final String URI = "uri";
+ private static final String HEADERS = "headers";
+ private static final String PROPERTIES = "properties";
+ private static final String BODY = "body";
+ private static final String PROVIDER = "provider";
+
+ private RemoteSignRequestParser() {}
+
+ public static String toJson(RemoteSignRequest request) {
+ return toJson(request, false);
+ }
+
+ public static String toJson(RemoteSignRequest request, boolean pretty) {
+ return JsonUtil.generate(gen -> toJson(request, gen), pretty);
+ }
+
+ public static void toJson(RemoteSignRequest request, JsonGenerator gen) throws IOException {
+ Preconditions.checkArgument(null != request, "Invalid remote sign request: null");
+
+ gen.writeStartObject();
+
+ gen.writeStringField(REGION, request.region());
+ gen.writeStringField(METHOD, request.method());
+ gen.writeStringField(URI, request.uri().toString());
+ headersToJson(HEADERS, request.headers(), gen);
+
+ if (!request.properties().isEmpty()) {
+ JsonUtil.writeStringMap(PROPERTIES, request.properties(), gen);
+ }
+
+ if (request.body() != null && !request.body().isEmpty()) {
+ gen.writeStringField(BODY, request.body());
+ }
+
+ if (request.provider() != null) {
+ gen.writeStringField(PROVIDER, request.provider());
+ }
+
+ gen.writeEndObject();
+ }
+
+ public static RemoteSignRequest fromJson(String json) {
+ return JsonUtil.parse(json, RemoteSignRequestParser::fromJson);
+ }
+
+ public static RemoteSignRequest fromJson(JsonNode json) {
+ Preconditions.checkArgument(null != json, "Cannot parse remote sign request from null object");
+ Preconditions.checkArgument(
+ json.isObject(), "Cannot parse remote sign request from non-object: %s", json);
+
+ String region = JsonUtil.getString(REGION, json);
+ String method = JsonUtil.getString(METHOD, json);
+ java.net.URI uri = java.net.URI.create(JsonUtil.getString(URI, json));
+ Map> headers = headersFromJson(HEADERS, json);
+
+ ImmutableRemoteSignRequest.Builder builder =
+ ImmutableRemoteSignRequest.builder()
+ .region(region)
+ .method(method)
+ .uri(uri)
+ .headers(headers);
+
+ if (json.has(PROPERTIES)) {
+ builder.properties(JsonUtil.getStringMap(PROPERTIES, json));
+ }
+
+ if (json.has(BODY)) {
+ builder.body(JsonUtil.getString(BODY, json));
+ }
+
+ if (json.has(PROVIDER)) {
+ builder.provider(JsonUtil.getString(PROVIDER, json));
+ }
+
+ return builder.build();
+ }
+
+ public static void headersToJson(
+ String property, Map> headers, JsonGenerator gen) throws IOException {
+ gen.writeObjectFieldStart(property);
+ for (Entry> entry : headers.entrySet()) {
+ gen.writeFieldName(entry.getKey());
+
+ gen.writeStartArray();
+ for (String val : entry.getValue()) {
+ gen.writeString(val);
+ }
+ gen.writeEndArray();
+ }
+ gen.writeEndObject();
+ }
+
+ public static Map> headersFromJson(String property, JsonNode json) {
+ Map> headers = Maps.newHashMap();
+ JsonNode headersNode = JsonUtil.get(property, json);
+ headersNode
+ .properties()
+ .forEach(
+ entry -> {
+ String key = entry.getKey();
+ List values = Arrays.asList(JsonUtil.getStringArray(entry.getValue()));
+ headers.put(key, values);
+ });
+ return headers;
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/ErrorResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/ErrorResponseParser.java
index 31ad0573b107..1329c074ab29 100644
--- a/core/src/main/java/org/apache/iceberg/rest/responses/ErrorResponseParser.java
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/ErrorResponseParser.java
@@ -46,9 +46,12 @@ public static String toJson(ErrorResponse errorResponse, boolean pretty) {
public static void toJson(ErrorResponse errorResponse, JsonGenerator generator)
throws IOException {
generator.writeStartObject();
+ writeError(errorResponse, generator);
+ generator.writeEndObject();
+ }
+ static void writeError(ErrorResponse errorResponse, JsonGenerator generator) throws IOException {
generator.writeObjectFieldStart(ERROR);
-
generator.writeStringField(MESSAGE, errorResponse.message());
generator.writeStringField(TYPE, errorResponse.type());
generator.writeNumberField(CODE, errorResponse.code());
@@ -57,8 +60,6 @@ public static void toJson(ErrorResponse errorResponse, JsonGenerator generator)
}
generator.writeEndObject();
-
- generator.writeEndObject();
}
/**
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponse.java
index 59db196244f5..2e176aac653f 100644
--- a/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponse.java
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponse.java
@@ -31,10 +31,12 @@
public class FetchPlanningResultResponse extends BaseScanTaskResponse {
private final PlanStatus planStatus;
+ private final ErrorResponse errorResponse;
private final List credentials;
private FetchPlanningResultResponse(
PlanStatus planStatus,
+ ErrorResponse errorResponse,
List planTasks,
List fileScanTasks,
List deleteFiles,
@@ -42,6 +44,7 @@ private FetchPlanningResultResponse(
List credentials) {
super(planTasks, fileScanTasks, deleteFiles, specsById);
this.planStatus = planStatus;
+ this.errorResponse = errorResponse;
this.credentials = credentials;
validate();
}
@@ -50,6 +53,10 @@ public PlanStatus planStatus() {
return planStatus;
}
+ public ErrorResponse errorResponse() {
+ return errorResponse;
+ }
+
public List credentials() {
return credentials != null ? credentials : ImmutableList.of();
}
@@ -64,6 +71,9 @@ public void validate() {
Preconditions.checkArgument(
planStatus() == PlanStatus.COMPLETED || (planTasks() == null && fileScanTasks() == null),
"Invalid response: tasks can only be returned in a 'completed' status");
+ Preconditions.checkArgument(
+ planStatus() == PlanStatus.FAILED || errorResponse() == null,
+ "Invalid response: error can only be returned in a 'failed' status");
if (fileScanTasks() == null || fileScanTasks().isEmpty()) {
Preconditions.checkArgument(
(deleteFiles() == null || deleteFiles().isEmpty()),
@@ -76,6 +86,7 @@ public static class Builder
private Builder() {}
private PlanStatus planStatus;
+ private ErrorResponse errorResponse;
private final List credentials = Lists.newArrayList();
public Builder withPlanStatus(PlanStatus status) {
@@ -83,6 +94,11 @@ public Builder withPlanStatus(PlanStatus status) {
return this;
}
+ public Builder withErrorResponse(ErrorResponse response) {
+ this.errorResponse = response;
+ return this;
+ }
+
public Builder withCredentials(List credentialsToAdd) {
credentials.addAll(credentialsToAdd);
return this;
@@ -91,7 +107,13 @@ public Builder withCredentials(List credentialsToAdd) {
@Override
public FetchPlanningResultResponse build() {
return new FetchPlanningResultResponse(
- planStatus, planTasks(), fileScanTasks(), deleteFiles(), specsById(), credentials);
+ planStatus,
+ errorResponse,
+ planTasks(),
+ fileScanTasks(),
+ deleteFiles(),
+ specsById(),
+ credentials);
}
}
}
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponseParser.java
index 4a523d3c023b..aa74049ab9f0 100644
--- a/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponseParser.java
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/FetchPlanningResultResponseParser.java
@@ -38,6 +38,7 @@ public class FetchPlanningResultResponseParser {
private static final String STATUS = "status";
private static final String PLAN_TASKS = "plan-tasks";
private static final String STORAGE_CREDENTIALS = "storage-credentials";
+ private static final String ERROR = "error";
private FetchPlanningResultResponseParser() {}
@@ -58,6 +59,11 @@ public static void toJson(FetchPlanningResultResponse response, JsonGenerator ge
"Cannot serialize fileScanTasks in fetchingPlanningResultResponse without specsById");
gen.writeStartObject();
gen.writeStringField(STATUS, response.planStatus().status());
+
+ if (response.errorResponse() != null) {
+ ErrorResponseParser.writeError(response.errorResponse(), gen);
+ }
+
if (response.planTasks() != null) {
JsonUtil.writeStringArray(PLAN_TASKS, response.planTasks(), gen);
}
@@ -90,6 +96,11 @@ public static FetchPlanningResultResponse fromJson(
json != null && !json.isEmpty(), "Invalid fetchPlanningResult response: null or empty");
PlanStatus planStatus = PlanStatus.fromName(JsonUtil.getString(STATUS, json));
+ ErrorResponse errorResponse = null;
+ if (json.has(ERROR) && json.get(ERROR).isObject()) {
+ errorResponse = ErrorResponseParser.fromJson(json);
+ }
+
List planTasks = JsonUtil.getStringListOrNull(PLAN_TASKS, json);
List deleteFiles = TableScanResponseParser.parseDeleteFiles(json, specsById);
List fileScanTasks =
@@ -98,6 +109,7 @@ public static FetchPlanningResultResponse fromJson(
FetchPlanningResultResponse.Builder builder =
FetchPlanningResultResponse.builder()
.withPlanStatus(planStatus)
+ .withErrorResponse(errorResponse)
.withPlanTasks(planTasks)
.withFileScanTasks(fileScanTasks)
.withSpecsById(specsById);
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponse.java
index 1b4bb86e65eb..d0ac222c3052 100644
--- a/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponse.java
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponse.java
@@ -33,11 +33,13 @@
public class PlanTableScanResponse extends BaseScanTaskResponse {
private final PlanStatus planStatus;
private final String planId;
+ private final ErrorResponse errorResponse;
private final List credentials;
private PlanTableScanResponse(
PlanStatus planStatus,
String planId,
+ ErrorResponse errorResponse,
List planTasks,
List fileScanTasks,
List deleteFiles,
@@ -46,6 +48,7 @@ private PlanTableScanResponse(
super(planTasks, fileScanTasks, deleteFiles, specsById);
this.planStatus = planStatus;
this.planId = planId;
+ this.errorResponse = errorResponse;
this.credentials = credentials;
validate();
}
@@ -58,6 +61,10 @@ public String planId() {
return planId;
}
+ public ErrorResponse errorResponse() {
+ return errorResponse;
+ }
+
public List credentials() {
return credentials != null ? credentials : ImmutableList.of();
}
@@ -86,6 +93,10 @@ public void validate() {
planStatus() == PlanStatus.COMPLETED || (planTasks() == null && fileScanTasks() == null),
"Invalid response: tasks can only be defined when status is '%s'",
PlanStatus.COMPLETED.status());
+ Preconditions.checkArgument(
+ planStatus() == PlanStatus.FAILED || errorResponse() == null,
+ "Invalid response: error can only be defined when status is '%s'",
+ PlanStatus.FAILED.status());
if (null != planId()) {
Preconditions.checkArgument(
planStatus() == PlanStatus.SUBMITTED || planStatus() == PlanStatus.COMPLETED,
@@ -108,6 +119,7 @@ public static Builder builder() {
public static class Builder extends BaseScanTaskResponse.Builder {
private PlanStatus planStatus;
private String planId;
+ private ErrorResponse errorResponse;
private final List credentials = Lists.newArrayList();
/**
@@ -127,6 +139,11 @@ public Builder withPlanId(String id) {
return this;
}
+ public Builder withErrorResponse(ErrorResponse response) {
+ this.errorResponse = response;
+ return this;
+ }
+
public Builder withCredentials(List credentialsToAdd) {
credentials.addAll(credentialsToAdd);
return this;
@@ -137,6 +154,7 @@ public PlanTableScanResponse build() {
return new PlanTableScanResponse(
planStatus,
planId,
+ errorResponse,
planTasks(),
fileScanTasks(),
deleteFiles(),
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponseParser.java
index c2f47b86d3f0..8ca199397ea6 100644
--- a/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponseParser.java
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/PlanTableScanResponseParser.java
@@ -39,6 +39,7 @@ public class PlanTableScanResponseParser {
private static final String PLAN_ID = "plan-id";
private static final String PLAN_TASKS = "plan-tasks";
private static final String STORAGE_CREDENTIALS = "storage-credentials";
+ private static final String ERROR = "error";
private PlanTableScanResponseParser() {}
@@ -60,6 +61,10 @@ public static void toJson(PlanTableScanResponse response, JsonGenerator gen) thr
gen.writeStartObject();
gen.writeStringField(STATUS, response.planStatus().status());
+ if (response.errorResponse() != null) {
+ ErrorResponseParser.writeError(response.errorResponse(), gen);
+ }
+
if (response.planId() != null) {
gen.writeStringField(PLAN_ID, response.planId());
}
@@ -98,6 +103,11 @@ public static PlanTableScanResponse fromJson(
"Cannot parse planTableScan response from empty or null object");
PlanStatus planStatus = PlanStatus.fromName(JsonUtil.getString(STATUS, json));
+ ErrorResponse errorResponse = null;
+ if (json.has(ERROR) && json.get(ERROR).isObject()) {
+ errorResponse = ErrorResponseParser.fromJson(json);
+ }
+
String planId = JsonUtil.getStringOrNull(PLAN_ID, json);
List planTasks = JsonUtil.getStringListOrNull(PLAN_TASKS, json);
List deleteFiles = TableScanResponseParser.parseDeleteFiles(json, specsById);
@@ -108,6 +118,7 @@ public static PlanTableScanResponse fromJson(
PlanTableScanResponse.builder()
.withPlanId(planId)
.withPlanStatus(planStatus)
+ .withErrorResponse(errorResponse)
.withPlanTasks(planTasks)
.withFileScanTasks(fileScanTasks)
.withSpecsById(specsById);
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponse.java
new file mode 100644
index 000000000000..c5009505bf4f
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponse.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest.responses;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.rest.RESTResponse;
+import org.immutables.value.Value;
+
+@Value.Immutable
+public interface RemoteSignResponse extends RESTResponse {
+ URI uri();
+
+ Map> headers();
+
+ @Override
+ default void validate() {}
+}
diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponseParser.java
new file mode 100644
index 000000000000..f53e844c6162
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/rest/responses/RemoteSignResponseParser.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest.responses;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.rest.requests.RemoteSignRequestParser;
+import org.apache.iceberg.util.JsonUtil;
+
+public class RemoteSignResponseParser {
+
+ private static final String URI_FIELD = "uri";
+ private static final String HEADERS = "headers";
+
+ private RemoteSignResponseParser() {}
+
+ public static String toJson(RemoteSignResponse response) {
+ return toJson(response, false);
+ }
+
+ public static String toJson(RemoteSignResponse response, boolean pretty) {
+ return JsonUtil.generate(gen -> toJson(response, gen), pretty);
+ }
+
+ public static void toJson(RemoteSignResponse response, JsonGenerator gen) throws IOException {
+ Preconditions.checkArgument(null != response, "Invalid remote sign response: null");
+
+ gen.writeStartObject();
+
+ gen.writeStringField(URI_FIELD, response.uri().toString());
+ RemoteSignRequestParser.headersToJson(HEADERS, response.headers(), gen);
+
+ gen.writeEndObject();
+ }
+
+ public static RemoteSignResponse fromJson(String json) {
+ return JsonUtil.parse(json, RemoteSignResponseParser::fromJson);
+ }
+
+ public static RemoteSignResponse fromJson(JsonNode json) {
+ Preconditions.checkArgument(null != json, "Cannot parse remote sign response from null object");
+ Preconditions.checkArgument(
+ json.isObject(), "Cannot parse remote sign response from non-object: %s", json);
+
+ URI uri = URI.create(JsonUtil.getString(URI_FIELD, json));
+ Map> headers = RemoteSignRequestParser.headersFromJson(HEADERS, json);
+
+ return ImmutableRemoteSignResponse.builder().uri(uri).headers(headers).build();
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/util/LocationUtil.java b/core/src/main/java/org/apache/iceberg/util/LocationUtil.java
index 400307149238..4c0d401c74b9 100644
--- a/core/src/main/java/org/apache/iceberg/util/LocationUtil.java
+++ b/core/src/main/java/org/apache/iceberg/util/LocationUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.iceberg.util;
+import java.util.UUID;
+import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Strings;
@@ -33,4 +35,26 @@ public static String stripTrailingSlash(String path) {
}
return result;
}
+
+ /**
+ * Returns a path component derived from the {@code tableIdentifier}, used as part of the table
+ * location URI.
+ *
+ * If {@code useUniqueLocation} is {@code true}, the returned component will include a random
+ * UUID suffix. Otherwise, the plain table name is returned.
+ *
+ * @param tableIdentifier Iceberg table identifier
+ * @param useUniqueLocation whether to ensure uniqueness
+ * @return a string representing the table name component for a location URI
+ */
+ public static String tableLocation(TableIdentifier tableIdentifier, boolean useUniqueLocation) {
+ Preconditions.checkArgument(null != tableIdentifier, "Invalid identifier: null");
+
+ if (useUniqueLocation) {
+ String uniqueSuffix = UUID.randomUUID().toString().replace("-", "");
+ return String.format("%s-%s", tableIdentifier.name(), uniqueSuffix);
+ } else {
+ return tableIdentifier.name();
+ }
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/util/LockManagers.java b/core/src/main/java/org/apache/iceberg/util/LockManagers.java
index 96622cb57f83..561d0a8975dd 100644
--- a/core/src/main/java/org/apache/iceberg/util/LockManagers.java
+++ b/core/src/main/java/org/apache/iceberg/util/LockManagers.java
@@ -18,11 +18,9 @@
*/
package org.apache.iceberg.util;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -108,6 +106,11 @@ public int heartbeatThreads() {
return heartbeatThreads;
}
+ /**
+ * Returns the shared scheduler for lock heartbeats.
+ *
+ *
Callers must not shut down this scheduler. It is shared across lock manager instances.
+ */
public ScheduledExecutorService scheduler() {
if (scheduler == null) {
synchronized (BaseLockManager.class) {
@@ -159,16 +162,10 @@ public void initialize(Map properties) {
@Override
public void close() throws Exception {
- if (scheduler != null) {
- List tasks = scheduler.shutdownNow();
- tasks.forEach(
- task -> {
- if (task instanceof Future) {
- ((Future) task).cancel(true);
- }
- });
- scheduler = null;
- }
+ // The scheduler is a shared static resource used across all BaseLockManager instances.
+ // Individual instances must not shut it down, as other instances may still be using it.
+ // The scheduler uses daemon threads and will be terminated at JVM exit by the shutdown
+ // hook registered via MoreExecutors.getExitingScheduledExecutorService.
}
}
diff --git a/core/src/main/java/org/apache/iceberg/util/StructLikeWrapper.java b/core/src/main/java/org/apache/iceberg/util/StructLikeWrapper.java
index 28629706bf5e..2e71d2419185 100644
--- a/core/src/main/java/org/apache/iceberg/util/StructLikeWrapper.java
+++ b/core/src/main/java/org/apache/iceberg/util/StructLikeWrapper.java
@@ -88,7 +88,13 @@ public boolean equals(Object other) {
return false;
}
- return comparator.compare(this.struct, that.struct) == 0;
+ try {
+ return comparator.compare(this.struct, that.struct) == 0;
+ } catch (RuntimeException e) {
+ // An exception may occur, for example, when struct is PartitionData and its type does not
+ // match its data.
+ return false;
+ }
}
@Override
diff --git a/core/src/test/java/org/apache/iceberg/stats/TestContentStats.java b/core/src/test/java/org/apache/iceberg/TestContentStats.java
similarity index 95%
rename from core/src/test/java/org/apache/iceberg/stats/TestContentStats.java
rename to core/src/test/java/org/apache/iceberg/TestContentStats.java
index 6baff7dfe63e..0f06276d454b 100644
--- a/core/src/test/java/org/apache/iceberg/stats/TestContentStats.java
+++ b/core/src/test/java/org/apache/iceberg/TestContentStats.java
@@ -16,22 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
-
-import static org.apache.iceberg.stats.FieldStatistic.AVG_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.EXACT_BOUNDS;
-import static org.apache.iceberg.stats.FieldStatistic.LOWER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.MAX_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.NAN_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.NULL_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.UPPER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.VALUE_COUNT;
+package org.apache.iceberg;
+
+import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.EXACT_BOUNDS;
+import static org.apache.iceberg.FieldStatistic.LOWER_BOUND;
+import static org.apache.iceberg.FieldStatistic.MAX_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.UPPER_BOUND;
+import static org.apache.iceberg.FieldStatistic.VALUE_COUNT;
import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.types.Types;
@@ -154,7 +153,7 @@ public void retrievalByPosition() {
assertThatThrownBy(() -> stats.get(0, Long.class))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining(
- "Wrong class, expected java.lang.Long but was org.apache.iceberg.stats.BaseFieldStats for object:");
+ "Wrong class, expected java.lang.Long but was org.apache.iceberg.BaseFieldStats for object:");
}
@Test
diff --git a/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java b/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
index 68e5fa8b560e..d7cdd5c5d884 100644
--- a/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
+++ b/core/src/test/java/org/apache/iceberg/TestDeleteFiles.java
@@ -615,6 +615,10 @@ public void removingDataFileByExpressionAlsoRemovesDV() {
.containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2");
assertThat(deleteSnap.deleteManifests(table.io())).hasSize(1);
+ assertThat(deleteSnap.summary())
+ .containsEntry(SnapshotSummary.REMOVED_DVS_PROP, "1")
+ .containsEntry(SnapshotSummary.REMOVED_DELETE_FILES_PROP, "1");
+
validateDeleteManifest(
deleteSnap.deleteManifests(table.io()).get(0),
dataSeqs(1L, 1L),
@@ -658,6 +662,10 @@ public void removingDataFileByPathAlsoRemovesDV() {
.containsEntry(SnapshotSummary.REPLACED_MANIFESTS_COUNT, "2");
assertThat(deleteSnap.deleteManifests(table.io())).hasSize(1);
+ assertThat(deleteSnap.summary())
+ .containsEntry(SnapshotSummary.REMOVED_DVS_PROP, "1")
+ .containsEntry(SnapshotSummary.REMOVED_DELETE_FILES_PROP, "1");
+
validateDeleteManifest(
deleteSnap.deleteManifests(table.io()).get(0),
dataSeqs(1L, 1L),
@@ -667,6 +675,69 @@ public void removingDataFileByPathAlsoRemovesDV() {
statuses(ManifestEntry.Status.DELETED, ManifestEntry.Status.EXISTING));
}
+ @TestTemplate
+ public void removingDataFilesWhenTruncatingAlsoRemovesDVs() {
+ assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
+ DeleteFile dv1 =
+ FileMetadata.deleteFileBuilder(SPEC)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-1-deletes.puffin")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=0")
+ .withRecordCount(5)
+ .withReferencedDataFile(DATA_FILE_BUCKET_0_IDS_0_2.location())
+ .withContentOffset(4)
+ .withContentSizeInBytes(6)
+ .build();
+
+ DeleteFile dv2 =
+ FileMetadata.deleteFileBuilder(SPEC)
+ .ofPositionDeletes()
+ .withPath("/path/to/data-2-deletes.puffin")
+ .withFileSizeInBytes(10)
+ .withPartitionPath("data_bucket=0")
+ .withRecordCount(5)
+ .withReferencedDataFile(DATA_FILE_BUCKET_0_IDS_8_10.location())
+ .withContentOffset(4)
+ .withContentSizeInBytes(6)
+ .build();
+
+ commit(
+ table,
+ table
+ .newRowDelta()
+ .addRows(DATA_FILE_BUCKET_0_IDS_0_2)
+ .addRows(DATA_FILE_BUCKET_0_IDS_8_10)
+ .addDeletes(dv1)
+ .addDeletes(dv2),
+ branch);
+
+ Snapshot snapshot = latestSnapshot(table, branch);
+ assertThat(snapshot.sequenceNumber()).isEqualTo(1);
+ assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(1);
+
+ // deleting by row filter should also remove the orphaned dv1 from delete manifests. When a
+ // table is truncated via TRUNCATE, the row filter is sent as Expressions.alwaysTrue()
+ commit(table, table.newDelete().deleteFromRowFilter(Expressions.alwaysTrue()), branch);
+
+ Snapshot deleteSnap = latestSnapshot(table, branch);
+ assertThat(deleteSnap.sequenceNumber()).isEqualTo(2);
+ assertThat(table.ops().current().lastSequenceNumber()).isEqualTo(2);
+
+ assertThat(deleteSnap.deleteManifests(table.io())).hasSize(1);
+ assertThat(deleteSnap.summary())
+ .containsEntry(SnapshotSummary.REMOVED_DVS_PROP, "2")
+ .containsEntry(SnapshotSummary.REMOVED_DELETE_FILES_PROP, "2");
+
+ validateDeleteManifest(
+ deleteSnap.deleteManifests(table.io()).get(0),
+ dataSeqs(1L, 1L),
+ fileSeqs(1L, 1L),
+ ids(deleteSnap.snapshotId(), deleteSnap.snapshotId()),
+ files(dv1, dv2),
+ statuses(Status.DELETED, Status.DELETED));
+ }
+
private static ByteBuffer longToBuffer(long value) {
return ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN).putLong(0, value);
}
diff --git a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
new file mode 100644
index 000000000000..325f9afd9ca9
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+class TestDeletionVectorStruct {
+
+ @Test
+ void testFieldAccess() {
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/data/dv.puffin")
+ .offset(256L)
+ .sizeInBytes(128L)
+ .cardinality(42L)
+ .build();
+
+ assertThat(dv.location()).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(dv.offset()).isEqualTo(256L);
+ assertThat(dv.sizeInBytes()).isEqualTo(128L);
+ assertThat(dv.cardinality()).isEqualTo(42L);
+ }
+
+ @Test
+ void testCopy() {
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/data/dv.puffin")
+ .offset(256L)
+ .sizeInBytes(128L)
+ .cardinality(42L)
+ .build();
+
+ DeletionVectorStruct copy = dv.copy();
+
+ assertThat(copy.location()).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(copy.offset()).isEqualTo(256L);
+ assertThat(copy.sizeInBytes()).isEqualTo(128L);
+ assertThat(copy.cardinality()).isEqualTo(42L);
+ }
+
+ @Test
+ void testSize() {
+ DeletionVectorStruct dv = new DeletionVectorStruct(DeletionVector.schema());
+ assertThat(dv.size()).isEqualTo(4);
+ }
+
+ @Test
+ void testProjectedStructLike() {
+ // project only location (field ID 155) and cardinality (field ID 156)
+ Types.StructType projection =
+ Types.StructType.of(DeletionVector.LOCATION, DeletionVector.CARDINALITY);
+
+ DeletionVectorStruct dv = new DeletionVectorStruct(projection);
+ assertThat(dv.size()).isEqualTo(2);
+
+ // projected position 0 maps to internal position 0 (location)
+ // projected position 1 maps to internal position 3 (cardinality)
+ dv.set(0, "s3://bucket/data/dv.puffin");
+ dv.set(1, 42L);
+
+ assertThat(dv.location()).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(dv.cardinality()).isEqualTo(42L);
+ assertThat(dv.get(0, String.class)).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(dv.get(1, Long.class)).isEqualTo(42L);
+ }
+
+ @Test
+ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException {
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/data/dv.puffin")
+ .offset(256L)
+ .sizeInBytes(128L)
+ .cardinality(42L)
+ .build();
+
+ DeletionVectorStruct deserialized = TestHelpers.roundTripSerialize(dv);
+
+ assertThat(deserialized.location()).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(deserialized.offset()).isEqualTo(256L);
+ assertThat(deserialized.sizeInBytes()).isEqualTo(128L);
+ assertThat(deserialized.cardinality()).isEqualTo(42L);
+ }
+
+ @Test
+ void testBuilderValidation() {
+ assertThatThrownBy(
+ () -> DeletionVectorStruct.builder().offset(0).sizeInBytes(1).cardinality(1).build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid location: null");
+
+ assertThatThrownBy(
+ () ->
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/dv.puffin")
+ .sizeInBytes(1)
+ .cardinality(1)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid offset: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/dv.puffin")
+ .offset(0)
+ .cardinality(1)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid size in bytes: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/dv.puffin")
+ .offset(0)
+ .sizeInBytes(1)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid cardinality: -1 (must be >= 0)");
+ }
+
+ @Test
+ void testKryoSerializationRoundTrip() throws IOException {
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/data/dv.puffin")
+ .offset(256L)
+ .sizeInBytes(128L)
+ .cardinality(42L)
+ .build();
+
+ DeletionVectorStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(dv);
+
+ assertThat(deserialized.location()).isEqualTo("s3://bucket/data/dv.puffin");
+ assertThat(deserialized.offset()).isEqualTo(256L);
+ assertThat(deserialized.sizeInBytes()).isEqualTo(128L);
+ assertThat(deserialized.cardinality()).isEqualTo(42L);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestEntryStatus.java b/core/src/test/java/org/apache/iceberg/TestEntryStatus.java
new file mode 100644
index 000000000000..c395cdcece7c
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestEntryStatus.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.stream.IntStream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class TestEntryStatus {
+
+ @ParameterizedTest
+ @EnumSource(EntryStatus.class)
+ void fromId(EntryStatus status) {
+ assertThat(EntryStatus.fromId(status.id())).isEqualTo(status);
+ }
+
+ static IntStream invalidIds() {
+ return IntStream.of(-1, EntryStatus.values().length);
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidIds")
+ void fromIdInvalid(int id) {
+ assertThatThrownBy(() -> EntryStatus.fromId(id))
+ .isInstanceOf(ArrayIndexOutOfBoundsException.class)
+ .hasMessageContaining(String.valueOf(id));
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/stats/TestFieldStats.java b/core/src/test/java/org/apache/iceberg/TestFieldStats.java
similarity index 94%
rename from core/src/test/java/org/apache/iceberg/stats/TestFieldStats.java
rename to core/src/test/java/org/apache/iceberg/TestFieldStats.java
index be5f3166940d..c703a3044fc0 100644
--- a/core/src/test/java/org/apache/iceberg/stats/TestFieldStats.java
+++ b/core/src/test/java/org/apache/iceberg/TestFieldStats.java
@@ -16,16 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
-
-import static org.apache.iceberg.stats.FieldStatistic.AVG_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.EXACT_BOUNDS;
-import static org.apache.iceberg.stats.FieldStatistic.LOWER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.MAX_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.NAN_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.NULL_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.UPPER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.VALUE_COUNT;
+package org.apache.iceberg;
+
+import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.EXACT_BOUNDS;
+import static org.apache.iceberg.FieldStatistic.LOWER_BOUND;
+import static org.apache.iceberg.FieldStatistic.MAX_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.UPPER_BOUND;
+import static org.apache.iceberg.FieldStatistic.VALUE_COUNT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -33,7 +33,6 @@
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.stream.Stream;
-import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
diff --git a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java
index 146f2c8da5e7..d665d84cad82 100644
--- a/core/src/test/java/org/apache/iceberg/TestLocationProvider.java
+++ b/core/src/test/java/org/apache/iceberg/TestLocationProvider.java
@@ -295,7 +295,7 @@ public void testExcludePartitionInPath() {
String fileLocation =
table.locationProvider().newDataLocation(table.spec(), partitionData, "test.parquet");
- // no partition values included in the path and last part of entropy is seperated with "-"
+ // no partition values included in the path and last part of entropy is separated with "-"
assertThat(fileLocation).endsWith("/data/0110/1010/0011/11101000-test.parquet");
}
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestEntryStatus.java b/core/src/test/java/org/apache/iceberg/TestManifestEntryStatus.java
new file mode 100644
index 000000000000..39867bbf7c02
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestManifestEntryStatus.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.stream.IntStream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class TestManifestEntryStatus {
+
+ @ParameterizedTest
+ @EnumSource(ManifestEntry.Status.class)
+ void fromId(ManifestEntry.Status status) {
+ assertThat(ManifestEntry.Status.fromId(status.id())).isEqualTo(status);
+ }
+
+ static IntStream invalidIds() {
+ return IntStream.of(-1, ManifestEntry.Status.values().length);
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidIds")
+ void fromIdInvalid(int id) {
+ assertThatThrownBy(() -> ManifestEntry.Status.fromId(id))
+ .isInstanceOf(ArrayIndexOutOfBoundsException.class)
+ .hasMessageContaining(String.valueOf(id));
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java b/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
new file mode 100644
index 000000000000..3a694f1a38f2
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestManifestInfoStruct.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+class TestManifestInfoStruct {
+
+ @Test
+ void testFieldAccess() {
+ ManifestInfoStruct info = new ManifestInfoStruct(ManifestInfo.schema());
+
+ info.set(0, 10);
+ info.set(1, 20);
+ info.set(2, 3);
+ info.set(3, 2);
+ info.set(4, 1000L);
+ info.set(5, 2000L);
+ info.set(6, 300L);
+ info.set(7, 200L);
+ info.set(8, 5L);
+ info.set(9, ByteBuffer.wrap(new byte[] {0xF}));
+ info.set(10, 1L);
+
+ assertThat(info.addedFilesCount()).isEqualTo(10);
+ assertThat(info.existingFilesCount()).isEqualTo(20);
+ assertThat(info.deletedFilesCount()).isEqualTo(3);
+ assertThat(info.replacedFilesCount()).isEqualTo(2);
+ assertThat(info.addedRowsCount()).isEqualTo(1000L);
+ assertThat(info.existingRowsCount()).isEqualTo(2000L);
+ assertThat(info.deletedRowsCount()).isEqualTo(300L);
+ assertThat(info.replacedRowsCount()).isEqualTo(200L);
+ assertThat(info.minSequenceNumber()).isEqualTo(5L);
+ assertThat(info.dv()).isNotNull();
+ assertThat(info.dvCardinality()).isEqualTo(1L);
+ }
+
+ @Test
+ void testCopy() {
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(10)
+ .existingFilesCount(20)
+ .deletedFilesCount(3)
+ .replacedFilesCount(2)
+ .addedRowsCount(1000L)
+ .existingRowsCount(2000L)
+ .deletedRowsCount(300L)
+ .replacedRowsCount(200L)
+ .minSequenceNumber(5L)
+ .dv(new byte[] {0xF})
+ .dvCardinality(1L)
+ .build();
+
+ ManifestInfoStruct copy = info.copy();
+
+ assertThat(copy.addedFilesCount()).isEqualTo(10);
+ assertThat(copy.existingFilesCount()).isEqualTo(20);
+ assertThat(copy.deletedFilesCount()).isEqualTo(3);
+ assertThat(copy.replacedFilesCount()).isEqualTo(2);
+ assertThat(copy.addedRowsCount()).isEqualTo(1000L);
+ assertThat(copy.existingRowsCount()).isEqualTo(2000L);
+ assertThat(copy.deletedRowsCount()).isEqualTo(300L);
+ assertThat(copy.replacedRowsCount()).isEqualTo(200L);
+ assertThat(copy.minSequenceNumber()).isEqualTo(5L);
+ assertThat(copy.dvCardinality()).isEqualTo(1L);
+
+ // verify deep copy of dv byte array
+ assertThat(copy.dv().array()).isNotSameAs(info.dv().array());
+ }
+
+ @Test
+ void testNullableFields() {
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build();
+
+ assertThat(info.dv()).isNull();
+ assertThat(info.dvCardinality()).isNull();
+ }
+
+ @Test
+ void testProjectedStructLike() {
+ // project only added_files_count (field ID 504) and min_sequence_number (field ID 516)
+ Types.StructType projection =
+ Types.StructType.of(ManifestInfo.ADDED_FILES_COUNT, ManifestInfo.MIN_SEQUENCE_NUMBER);
+
+ ManifestInfoStruct info = new ManifestInfoStruct(projection);
+ assertThat(info.size()).isEqualTo(2);
+
+ // projected position 0 maps to internal position 0 (added_files_count)
+ // projected position 1 maps to internal position 8 (min_sequence_number)
+ info.set(0, 10);
+ info.set(1, 5L);
+
+ assertThat(info.addedFilesCount()).isEqualTo(10);
+ assertThat(info.minSequenceNumber()).isEqualTo(5L);
+ assertThat(info.get(0, Integer.class)).isEqualTo(10);
+ assertThat(info.get(1, Long.class)).isEqualTo(5L);
+ }
+
+ @Test
+ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException {
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(10)
+ .existingFilesCount(20)
+ .deletedFilesCount(3)
+ .replacedFilesCount(2)
+ .addedRowsCount(1000L)
+ .existingRowsCount(2000L)
+ .deletedRowsCount(300L)
+ .replacedRowsCount(200L)
+ .minSequenceNumber(5L)
+ .dv(new byte[] {0xF})
+ .dvCardinality(1L)
+ .build();
+
+ ManifestInfoStruct deserialized = TestHelpers.roundTripSerialize(info);
+
+ assertThat(deserialized.addedFilesCount()).isEqualTo(10);
+ assertThat(deserialized.existingFilesCount()).isEqualTo(20);
+ assertThat(deserialized.deletedFilesCount()).isEqualTo(3);
+ assertThat(deserialized.replacedFilesCount()).isEqualTo(2);
+ assertThat(deserialized.addedRowsCount()).isEqualTo(1000L);
+ assertThat(deserialized.existingRowsCount()).isEqualTo(2000L);
+ assertThat(deserialized.deletedRowsCount()).isEqualTo(300L);
+ assertThat(deserialized.replacedRowsCount()).isEqualTo(200L);
+ assertThat(deserialized.minSequenceNumber()).isEqualTo(5L);
+ assertThat(deserialized.dv()).isEqualTo(ByteBuffer.wrap(new byte[] {0xF}));
+ assertThat(deserialized.dvCardinality()).isEqualTo(1L);
+ }
+
+ @Test
+ void testBuilderValidation() {
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid added files count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid existing files count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid deleted files count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid replaced files count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid added rows count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid existing rows count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid deleted rows count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid replaced rows count: -1 (must be >= 0)");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid min sequence number: -1 (must be >= 0)");
+ }
+
+ @Test
+ void testBuilderDvPairingValidation() {
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .dv(new byte[] {0xF})
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid DV and cardinality: must both be null or non-null");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .dvCardinality(1L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid DV and cardinality: must both be null or non-null");
+
+ assertThatThrownBy(
+ () ->
+ ManifestInfoStruct.builder()
+ .addedFilesCount(0)
+ .existingFilesCount(0)
+ .deletedFilesCount(0)
+ .replacedFilesCount(0)
+ .addedRowsCount(0L)
+ .existingRowsCount(0L)
+ .deletedRowsCount(0L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(0L)
+ .dv(new byte[] {0xF})
+ .dvCardinality(0L)
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid DV cardinality: 0 (must be positive)");
+ }
+
+ @Test
+ void testKryoSerializationRoundTrip() throws IOException {
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(10)
+ .existingFilesCount(20)
+ .deletedFilesCount(3)
+ .replacedFilesCount(2)
+ .addedRowsCount(1000L)
+ .existingRowsCount(2000L)
+ .deletedRowsCount(300L)
+ .replacedRowsCount(200L)
+ .minSequenceNumber(5L)
+ .dv(new byte[] {0xF})
+ .dvCardinality(1L)
+ .build();
+
+ ManifestInfoStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(info);
+
+ assertThat(deserialized.addedFilesCount()).isEqualTo(10);
+ assertThat(deserialized.existingFilesCount()).isEqualTo(20);
+ assertThat(deserialized.deletedFilesCount()).isEqualTo(3);
+ assertThat(deserialized.replacedFilesCount()).isEqualTo(2);
+ assertThat(deserialized.addedRowsCount()).isEqualTo(1000L);
+ assertThat(deserialized.existingRowsCount()).isEqualTo(2000L);
+ assertThat(deserialized.deletedRowsCount()).isEqualTo(300L);
+ assertThat(deserialized.replacedRowsCount()).isEqualTo(200L);
+ assertThat(deserialized.minSequenceNumber()).isEqualTo(5L);
+ assertThat(deserialized.dv()).isEqualTo(ByteBuffer.wrap(new byte[] {0xF}));
+ assertThat(deserialized.dvCardinality()).isEqualTo(1L);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index d9048a5d5ed3..874bb6d74d44 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -41,8 +41,6 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.stats.ContentStats;
-import org.apache.iceberg.stats.FieldStats;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.types.Types.BinaryType;
diff --git a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
index 384b7132ef76..09e9fdd1f722 100644
--- a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
+++ b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
@@ -2207,4 +2207,47 @@ private static PartitionStatisticsFile reusePartitionStatsFile(
private static void commitPartitionStats(Table table, PartitionStatisticsFile statisticsFile) {
table.updatePartitionStatistics().setPartitionStatistics(statisticsFile).commit();
}
+
+ @TestTemplate
+ public void testAppendOnlyManifestsNotScannedDuringCleanup() {
+ assumeThat(incrementalCleanup).isTrue();
+
+ TestTables.LocalFileIO spyFileIO = Mockito.spy(new TestTables.LocalFileIO());
+ String tableName = "testAppendOnlyManifests";
+ Table testTable =
+ TestTables.create(
+ tableDir,
+ tableName,
+ SCHEMA,
+ SPEC,
+ SortOrder.unsorted(),
+ formatVersion,
+ new TestTables.TestTableOperations(tableName, tableDir, spyFileIO));
+
+ testTable.newAppend().appendFile(FILE_A).commit();
+ Snapshot firstSnapshot = testTable.currentSnapshot();
+
+ Set appendOnlyManifestPaths =
+ firstSnapshot.allManifests(testTable.io()).stream()
+ .map(ManifestFile::path)
+ .collect(Collectors.toSet());
+
+ waitUntilAfter(firstSnapshot.timestampMillis());
+
+ testTable.newAppend().appendFile(FILE_B).commit();
+ long tAfterCommits = waitUntilAfter(testTable.currentSnapshot().timestampMillis());
+
+ Mockito.clearInvocations(spyFileIO);
+
+ Set deletedFiles = Sets.newHashSet();
+ removeSnapshots(testTable)
+ .expireOlderThan(tAfterCommits)
+ .deleteWith(deletedFiles::add)
+ .commit();
+
+ assertThat(deletedFiles).containsExactly(firstSnapshot.manifestListLocation());
+
+ appendOnlyManifestPaths.forEach(
+ path -> Mockito.verify(spyFileIO, Mockito.never()).newInputFile(path));
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/TestRowDelta.java b/core/src/test/java/org/apache/iceberg/TestRowDelta.java
index c442541289a9..aaccf4122481 100644
--- a/core/src/test/java/org/apache/iceberg/TestRowDelta.java
+++ b/core/src/test/java/org/apache/iceberg/TestRowDelta.java
@@ -2419,6 +2419,27 @@ public void testManifestMergingAfterUpgradeToV3() {
assertThat(taskDV.contentSizeInBytes()).isEqualTo(dv.contentSizeInBytes());
}
+ @TestTemplate
+ public void testV2StagedPositionDeleteCannotCommitToV3() {
+ assumeThat(formatVersion).isEqualTo(2);
+
+ Snapshot initial = commit(table, table.newAppend().appendFile(FILE_A), branch);
+
+ // Stage RowDelta at v2: position delete for FILE_A + add new data FILE_B.
+ RowDelta rowDelta = table.newRowDelta().addDeletes(FILE_A_DELETES).addRows(FILE_B);
+
+ // upgrade the table
+ table.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit();
+
+ assertThatThrownBy(rowDelta::commit)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Must use DVs for position deletes in V3");
+
+ table.refresh();
+ assertThat(table.operations().current().formatVersion()).isEqualTo(3);
+ assertThat(table.snapshot(branch)).isEqualTo(initial);
+ }
+
@TestTemplate
public void testInabilityToAddPositionDeleteFilesInTablesWithDVs() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(3);
diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java
index fb942dde2aa2..5325e4013c68 100644
--- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java
+++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java
@@ -1181,6 +1181,17 @@ public void testDeleteMapKey() {
.hasMessageStartingWith("Cannot delete map keys");
}
+ @Test
+ public void testDeleteMapValue() {
+ assertThatThrownBy(
+ () ->
+ new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID)
+ .deleteColumn("locations.value")
+ .apply())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageStartingWith("Cannot delete value type from map");
+ }
+
@Test
public void testAddFieldToMapKey() {
assertThatThrownBy(
diff --git a/api/src/test/java/org/apache/iceberg/stats/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
similarity index 95%
rename from api/src/test/java/org/apache/iceberg/stats/TestStatsUtil.java
rename to core/src/test/java/org/apache/iceberg/TestStatsUtil.java
index 62c7c0ea75fb..54db9e5d2095 100644
--- a/api/src/test/java/org/apache/iceberg/stats/TestStatsUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
@@ -16,23 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.stats;
+package org.apache.iceberg;
-import static org.apache.iceberg.stats.FieldStatistic.AVG_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.EXACT_BOUNDS;
-import static org.apache.iceberg.stats.FieldStatistic.LOWER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.MAX_VALUE_SIZE;
-import static org.apache.iceberg.stats.FieldStatistic.NAN_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.NULL_VALUE_COUNT;
-import static org.apache.iceberg.stats.FieldStatistic.UPPER_BOUND;
-import static org.apache.iceberg.stats.FieldStatistic.VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.EXACT_BOUNDS;
+import static org.apache.iceberg.FieldStatistic.LOWER_BOUND;
+import static org.apache.iceberg.FieldStatistic.MAX_VALUE_SIZE;
+import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT;
+import static org.apache.iceberg.FieldStatistic.UPPER_BOUND;
+import static org.apache.iceberg.FieldStatistic.VALUE_COUNT;
import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
new file mode 100644
index 000000000000..6d84fd542345
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+public class TestTrackedFile {
+
+ private static final Schema TABLE_SCHEMA =
+ new Schema(
+ optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get()));
+
+ private static final Types.StructType CONTENT_STATS_TYPE =
+ StatsUtil.contentStatsFor(TABLE_SCHEMA).type().asStructType();
+
+ @Test
+ public void schemaWithContentStatsFieldOrder() {
+ Types.StructType type = TrackedFile.schemaWithContentStats(CONTENT_STATS_TYPE);
+ List fields = type.fields();
+
+ assertThat(fields)
+ .extracting(Types.NestedField::name)
+ .containsExactly(
+ "tracking",
+ "content_type",
+ "location",
+ "file_format",
+ "record_count",
+ "file_size_in_bytes",
+ "spec_id",
+ "content_stats",
+ "sort_order_id",
+ "deletion_vector",
+ "manifest_info",
+ "key_metadata",
+ "split_offsets",
+ "equality_ids");
+ }
+
+ @Test
+ public void schemaWithContentStatsFieldIds() {
+ Types.StructType type = TrackedFile.schemaWithContentStats(CONTENT_STATS_TYPE);
+ List fields = type.fields();
+
+ assertThat(fields)
+ .extracting(Types.NestedField::fieldId)
+ .containsExactly(147, 134, 100, 101, 103, 104, 141, 146, 140, 148, 150, 131, 132, 135);
+ }
+
+ @Test
+ public void schemaWithContentStatsUsesProvidedType() {
+ Types.StructType type = TrackedFile.schemaWithContentStats(CONTENT_STATS_TYPE);
+ Types.NestedField contentStatsField = type.field(TrackedFile.CONTENT_STATS_ID);
+
+ assertThat(contentStatsField.type().asStructType()).isEqualTo(CONTENT_STATS_TYPE);
+ }
+
+ @Test
+ public void schemaWithContentStatsReflectsInput() {
+ Schema smallSchema = new Schema(optional(1, "id", Types.IntegerType.get()));
+ Schema largeSchema =
+ new Schema(
+ optional(1, "id", Types.IntegerType.get()),
+ optional(2, "data", Types.StringType.get()),
+ optional(3, "ts", Types.TimestampType.withoutZone()));
+
+ Types.StructType smallStats = StatsUtil.contentStatsFor(smallSchema).type().asStructType();
+ Types.StructType largeStats = StatsUtil.contentStatsFor(largeSchema).type().asStructType();
+
+ Types.StructType smallType = TrackedFile.schemaWithContentStats(smallStats);
+ Types.StructType largeType = TrackedFile.schemaWithContentStats(largeStats);
+
+ Types.StructType smallResult =
+ smallType.field(TrackedFile.CONTENT_STATS_ID).type().asStructType();
+ Types.StructType largeResult =
+ largeType.field(TrackedFile.CONTENT_STATS_ID).type().asStructType();
+
+ assertThat(smallResult.fields()).hasSize(1);
+ assertThat(largeResult.fields()).hasSize(3);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
new file mode 100644
index 000000000000..62324e5607ef
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+
+class TestTrackedFileStruct {
+ @Test
+ void testFieldAccess() {
+ TrackedFileStruct file = new TrackedFileStruct();
+ TrackingStruct tracking =
+ TrackingStruct.builder().status(EntryStatus.ADDED).snapshotId(42L).build();
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/dv.puffin")
+ .offset(100L)
+ .sizeInBytes(50L)
+ .cardinality(5L)
+ .build();
+ ManifestInfoStruct info =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(10)
+ .existingFilesCount(20)
+ .deletedFilesCount(3)
+ .replacedFilesCount(2)
+ .addedRowsCount(1000L)
+ .existingRowsCount(2000L)
+ .deletedRowsCount(300L)
+ .replacedRowsCount(200L)
+ .minSequenceNumber(5L)
+ .build();
+
+ file.set(0, tracking);
+ file.set(1, FileContent.EQUALITY_DELETES.id());
+ file.set(2, "s3://bucket/data/eq-delete.avro");
+ file.set(3, "avro");
+ file.set(4, 50L);
+ file.set(5, 512L);
+ file.set(6, 1);
+ file.set(8, 5);
+ file.set(9, dv);
+ file.set(10, info);
+ file.set(11, ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ file.set(12, ImmutableList.of(100L, 200L));
+ file.set(13, ImmutableList.of(1, 2, 3));
+
+ assertThat(file.tracking()).isNotNull();
+ assertThat(file.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(file.tracking().snapshotId()).isEqualTo(42L);
+ assertThat(file.contentType()).isEqualTo(FileContent.EQUALITY_DELETES);
+ assertThat(file.location()).isEqualTo("s3://bucket/data/eq-delete.avro");
+ assertThat(file.fileFormat()).isEqualTo(FileFormat.AVRO);
+ assertThat(file.recordCount()).isEqualTo(50L);
+ assertThat(file.fileSizeInBytes()).isEqualTo(512L);
+ assertThat(file.specId()).isEqualTo(1);
+ assertThat(file.sortOrderId()).isEqualTo(5);
+ assertThat(file.deletionVector()).isSameAs(dv);
+ assertThat(file.manifestInfo()).isSameAs(info);
+ assertThat(file.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ assertThat(file.splitOffsets()).containsExactly(100L, 200L);
+ assertThat(file.equalityIds()).containsExactly(1, 2, 3);
+ }
+
+ @Test
+ void testReaderSideFields() {
+ TrackedFileStruct file = new TrackedFileStruct();
+
+ TrackingStruct tracking = TrackingStruct.builder().status(EntryStatus.ADDED).build();
+ tracking.setManifestLocation("s3://bucket/metadata/manifest.avro");
+ tracking.set(8, 7L);
+
+ file.set(0, tracking);
+ file.set(1, FileContent.DATA.id());
+ file.set(2, "test");
+ file.set(3, "parquet");
+ file.set(4, 0L);
+ file.set(5, 0L);
+
+ assertThat(file.tracking().manifestLocation()).isEqualTo("s3://bucket/metadata/manifest.avro");
+ assertThat(file.tracking().manifestPos()).isEqualTo(7L);
+ }
+
+ @Test
+ void testCopy() {
+ TrackedFileStruct file = createFullTrackedFile();
+
+ TrackedFile copy = file.copy();
+ assertThat(copy).isInstanceOf(TrackedFileStruct.class);
+
+ assertThat(copy.contentType()).isEqualTo(FileContent.DATA);
+ assertThat(copy.location()).isEqualTo("s3://bucket/data/file.parquet");
+ assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET);
+ assertThat(copy.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(copy.tracking().snapshotId()).isEqualTo(42L);
+ assertThat(copy.deletionVector().location()).isEqualTo("s3://bucket/dv.puffin");
+ assertThat(copy.specId()).isEqualTo(0);
+ assertThat(copy.sortOrderId()).isEqualTo(1);
+ assertThat(copy.recordCount()).isEqualTo(100L);
+ assertThat(copy.fileSizeInBytes()).isEqualTo(1024L);
+ assertThat(copy.keyMetadata()).isNotNull();
+ assertThat(copy.splitOffsets()).containsExactly(50L);
+ assertThat(copy.equalityIds()).isNull();
+ assertThat(copy.tracking().manifestLocation()).isEqualTo("s3://bucket/manifest.avro");
+ assertThat(copy.tracking().manifestPos()).isEqualTo(3L);
+ }
+
+ @Test
+ void testCopyWithoutStats() {
+ TrackedFileStruct file = createTrackedFileWithStats();
+ assertThat(file.contentStats()).isNotNull();
+
+ TrackedFile copy = file.copyWithoutStats();
+
+ assertThat(copy.contentType()).isEqualTo(FileContent.DATA);
+ assertThat(copy.location()).isEqualTo("s3://bucket/data/file.parquet");
+ assertThat(copy.contentStats()).isNull();
+ }
+
+ @Test
+ void testCopyWithStatsFilters() {
+ TrackedFileStruct file = createTrackedFileWithStats();
+ Set keepFieldIds = ImmutableSet.of(1);
+
+ TrackedFile copy = file.copyWithStats(keepFieldIds);
+
+ assertThat(copy.contentStats()).isNotNull();
+ ContentStats stats = copy.contentStats();
+ assertThat(stats.fieldStats()).hasSize(1);
+ assertThat(stats.fieldStats().get(0).fieldId()).isEqualTo(1);
+ }
+
+ @Test
+ void testCopyIsDeep() {
+ TrackedFileStruct file = createFullTrackedFile();
+
+ TrackedFile copy = file.copy();
+
+ // keyMetadata should be a deep copy
+ assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata());
+ }
+
+ @Test
+ void testStructLikeSize() {
+ TrackedFileStruct file = new TrackedFileStruct();
+ assertThat(file.size()).isEqualTo(14);
+ }
+
+ @Test
+ void testStructLikeGetSet() {
+ TrackedFileStruct file = new TrackedFileStruct();
+
+ file.set(1, FileContent.DATA.id());
+ assertThat(file.get(1, Integer.class)).isEqualTo(FileContent.DATA.id());
+
+ file.set(2, "test-location");
+ assertThat(file.get(2, String.class)).isEqualTo("test-location");
+
+ file.set(4, 999L);
+ assertThat(file.get(4, Long.class)).isEqualTo(999L);
+ }
+
+ @Test
+ void testProjectedStructLike() {
+ // project only location (field ID 100) and file_size_in_bytes (field ID 104)
+ Types.StructType projection =
+ Types.StructType.of(TrackedFile.LOCATION, TrackedFile.FILE_SIZE_IN_BYTES);
+
+ TrackedFileStruct file = new TrackedFileStruct(projection);
+ assertThat(file.size()).isEqualTo(2);
+
+ // projected position 0 maps to internal position 2 (location)
+ // projected position 1 maps to internal position 5 (file_size_in_bytes)
+ file.set(0, "s3://bucket/file.parquet");
+ file.set(1, 1024L);
+
+ assertThat(file.location()).isEqualTo("s3://bucket/file.parquet");
+ assertThat(file.fileSizeInBytes()).isEqualTo(1024L);
+ assertThat(file.get(0, String.class)).isEqualTo("s3://bucket/file.parquet");
+ assertThat(file.get(1, Long.class)).isEqualTo(1024L);
+ }
+
+ @Test
+ void testContentStatsReturnedWhenPresent() {
+ TrackedFileStruct file = createTrackedFileWithStats();
+ assertThat(file.contentStats()).isNotNull();
+ assertThat(file.contentStats().fieldStats()).hasSize(2);
+ }
+
+ @Test
+ void testContentStatsNullWhenNotSet() {
+ TrackedFileStruct file = new TrackedFileStruct();
+ file.set(1, FileContent.DATA.id());
+ file.set(2, "test");
+ file.set(3, "parquet");
+ file.set(4, 0L);
+ file.set(5, 0L);
+ file.set(6, 0);
+
+ assertThat(file.contentStats()).isNull();
+ }
+
+ @Test
+ void testAllFileContentTypesSupported() {
+ for (FileContent content : FileContent.values()) {
+ TrackedFileStruct file = new TrackedFileStruct();
+ file.set(1, content.id());
+ assertThat(file.contentType()).isEqualTo(content);
+ }
+ }
+
+ @Test
+ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException {
+ TrackedFileStruct file = createFullTrackedFile();
+
+ TrackedFileStruct deserialized = TestHelpers.roundTripSerialize(file);
+
+ assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA);
+ assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet");
+ assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET);
+ assertThat(deserialized.recordCount()).isEqualTo(100L);
+ assertThat(deserialized.fileSizeInBytes()).isEqualTo(1024L);
+ assertThat(deserialized.specId()).isEqualTo(0);
+ assertThat(deserialized.sortOrderId()).isEqualTo(1);
+ assertThat(deserialized.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(deserialized.tracking().snapshotId()).isEqualTo(42L);
+ assertThat(deserialized.deletionVector().location()).isEqualTo("s3://bucket/dv.puffin");
+ assertThat(deserialized.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ assertThat(deserialized.splitOffsets()).containsExactly(50L);
+ assertThat(deserialized.tracking().manifestPos()).isEqualTo(3L);
+ assertThat(deserialized.tracking().manifestLocation()).isEqualTo("s3://bucket/manifest.avro");
+ }
+
+ @Test
+ void testKryoSerializationRoundTrip() throws IOException {
+ TrackedFileStruct file = createFullTrackedFile();
+
+ TrackedFileStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(file);
+
+ assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA);
+ assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet");
+ assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET);
+ assertThat(deserialized.recordCount()).isEqualTo(100L);
+ assertThat(deserialized.fileSizeInBytes()).isEqualTo(1024L);
+ assertThat(deserialized.specId()).isEqualTo(0);
+ assertThat(deserialized.sortOrderId()).isEqualTo(1);
+ assertThat(deserialized.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(deserialized.tracking().snapshotId()).isEqualTo(42L);
+ assertThat(deserialized.deletionVector().location()).isEqualTo("s3://bucket/dv.puffin");
+ assertThat(deserialized.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ assertThat(deserialized.splitOffsets()).containsExactly(50L);
+ assertThat(deserialized.tracking().manifestPos()).isEqualTo(3L);
+ assertThat(deserialized.tracking().manifestLocation()).isEqualTo("s3://bucket/manifest.avro");
+ }
+
+ static TrackedFileStruct createFullTrackedFile() {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(42L)
+ .dataSequenceNumber(10L)
+ .build();
+ tracking.setManifestLocation("s3://bucket/manifest.avro");
+ tracking.set(8, 3L);
+
+ DeletionVectorStruct dv =
+ DeletionVectorStruct.builder()
+ .location("s3://bucket/dv.puffin")
+ .offset(100L)
+ .sizeInBytes(50L)
+ .cardinality(5L)
+ .build();
+
+ TrackedFileStruct file =
+ new TrackedFileStruct(
+ tracking,
+ FileContent.DATA,
+ "s3://bucket/data/file.parquet",
+ FileFormat.PARQUET,
+ 100L,
+ 1024L);
+ file.set(6, 0);
+ file.set(8, 1);
+ file.set(9, dv);
+ file.set(11, ByteBuffer.wrap(new byte[] {1, 2, 3}));
+ file.set(12, ImmutableList.of(50L));
+
+ return file;
+ }
+
+ @SuppressWarnings("unchecked")
+ static TrackedFileStruct createTrackedFileWithStats() {
+ Types.StructType statsStruct =
+ Types.StructType.of(
+ Types.NestedField.optional(
+ 10000,
+ "1",
+ Types.StructType.of(
+ Types.NestedField.optional(10001, "value_count", Types.LongType.get()),
+ Types.NestedField.optional(10002, "null_value_count", Types.LongType.get()),
+ Types.NestedField.optional(10003, "nan_value_count", Types.LongType.get()),
+ Types.NestedField.optional(10006, "lower_bound", Types.IntegerType.get()),
+ Types.NestedField.optional(10007, "upper_bound", Types.IntegerType.get()))),
+ Types.NestedField.optional(
+ 20000,
+ "2",
+ Types.StructType.of(
+ Types.NestedField.optional(20001, "value_count", Types.LongType.get()),
+ Types.NestedField.optional(20002, "null_value_count", Types.LongType.get()),
+ Types.NestedField.optional(20003, "nan_value_count", Types.LongType.get()),
+ Types.NestedField.optional(20006, "lower_bound", Types.FloatType.get()),
+ Types.NestedField.optional(20007, "upper_bound", Types.FloatType.get()))));
+
+ List> fieldStatsList =
+ ImmutableList.of(
+ (FieldStats)
+ BaseFieldStats.builder()
+ .fieldId(1)
+ .type(Types.IntegerType.get())
+ .valueCount(100L)
+ .nullValueCount(5L)
+ .lowerBound(1)
+ .upperBound(1000)
+ .build(),
+ (FieldStats)
+ BaseFieldStats.builder()
+ .fieldId(2)
+ .type(Types.FloatType.get())
+ .valueCount(200L)
+ .nullValueCount(10L)
+ .nanValueCount(3L)
+ .lowerBound(1.0f)
+ .upperBound(100.0f)
+ .build());
+
+ BaseContentStats stats =
+ BaseContentStats.builder()
+ .withStatsStruct(statsStruct)
+ .withFieldStats(fieldStatsList)
+ .build();
+
+ TrackedFileStruct file =
+ new TrackedFileStruct(
+ null,
+ FileContent.DATA,
+ "s3://bucket/data/file.parquet",
+ FileFormat.PARQUET,
+ 100L,
+ 1024L);
+ file.set(6, 0);
+ file.set(7, stats);
+
+ return file;
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java
new file mode 100644
index 000000000000..98a7eff2af45
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+class TestTrackingStruct {
+
+ @Test
+ void testFieldAccess() {
+ TrackingStruct tracking = new TrackingStruct(Tracking.schema());
+
+ tracking.set(0, EntryStatus.ADDED.id());
+ tracking.set(1, 42L);
+ tracking.set(2, 10L);
+ tracking.set(3, 11L);
+ tracking.set(4, 43L);
+ tracking.set(5, 1000L);
+
+ assertThat(tracking.status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(tracking.snapshotId()).isEqualTo(42L);
+ assertThat(tracking.dataSequenceNumber()).isEqualTo(10L);
+ assertThat(tracking.fileSequenceNumber()).isEqualTo(11L);
+ assertThat(tracking.dvSnapshotId()).isEqualTo(43L);
+ assertThat(tracking.firstRowId()).isEqualTo(1000L);
+ assertThat(tracking.deletedPositions()).isNull();
+ assertThat(tracking.replacedPositions()).isNull();
+ }
+
+ @Test
+ void testCopy() {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(42L)
+ .dataSequenceNumber(10L)
+ .deletedPositions(new byte[] {1, 2})
+ .build();
+
+ TrackingStruct copy = tracking.copy();
+
+ assertThat(copy.status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(copy.snapshotId()).isEqualTo(42L);
+ assertThat(copy.dataSequenceNumber()).isEqualTo(10L);
+ assertThat(copy.deletedPositions()).isNotNull();
+
+ // verify deep copy of ByteBuffer
+ assertThat(copy.deletedPositions()).isNotSameAs(tracking.deletedPositions());
+ }
+
+ @ParameterizedTest
+ @EnumSource(EntryStatus.class)
+ void testAllStatuses(EntryStatus status) {
+ TrackingStruct tracking = new TrackingStruct(Tracking.schema());
+ tracking.set(0, status.id());
+ assertThat(tracking.status()).isEqualTo(status);
+ }
+
+ @Test
+ void testIsLive() {
+ TrackingStruct tracking = new TrackingStruct(Tracking.schema());
+
+ tracking.set(0, EntryStatus.ADDED.id());
+ assertThat(tracking.isLive()).isTrue();
+
+ tracking.set(0, EntryStatus.EXISTING.id());
+ assertThat(tracking.isLive()).isTrue();
+
+ tracking.set(0, EntryStatus.DELETED.id());
+ assertThat(tracking.isLive()).isFalse();
+
+ tracking.set(0, EntryStatus.REPLACED.id());
+ assertThat(tracking.isLive()).isFalse();
+ }
+
+ @Test
+ void testInheritSnapshotId() {
+ TrackingStruct tracking = TrackingStruct.builder().status(EntryStatus.ADDED).build();
+ tracking.inheritFrom(createManifestTracking(100L, 60L));
+
+ // snapshotId is null, should inherit from manifest
+ assertThat(tracking.snapshotId()).isEqualTo(100L);
+ }
+
+ @Test
+ void testInheritSequenceNumberForAddedEntries() {
+ TrackingStruct tracking = TrackingStruct.builder().status(EntryStatus.ADDED).build();
+ tracking.inheritFrom(createManifestTracking(100L, 60L));
+
+ // sequence numbers are null and status is ADDED, should inherit
+ assertThat(tracking.dataSequenceNumber()).isEqualTo(60L);
+ assertThat(tracking.fileSequenceNumber()).isEqualTo(60L);
+ }
+
+ @Test
+ void testDoNotInheritSequenceNumberForExistingEntries() {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.EXISTING)
+ .dataSequenceNumber(5L)
+ .fileSequenceNumber(6L)
+ .build();
+ tracking.inheritFrom(createManifestTracking(100L, 60L));
+
+ // sequence numbers are not inherited for EXISTING entries
+ assertThat(tracking.dataSequenceNumber()).isEqualTo(5L);
+ assertThat(tracking.fileSequenceNumber()).isEqualTo(6L);
+ }
+
+ @Test
+ void testExplicitValuesOverrideInheritance() {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(200L)
+ .dataSequenceNumber(75L)
+ .fileSequenceNumber(76L)
+ .build();
+ tracking.inheritFrom(createManifestTracking(100L, 60L));
+
+ // explicit values should take precedence
+ assertThat(tracking.snapshotId()).isEqualTo(200L);
+ assertThat(tracking.dataSequenceNumber()).isEqualTo(75L);
+ assertThat(tracking.fileSequenceNumber()).isEqualTo(76L);
+ }
+
+ @Test
+ void testInheritFromRejectsUnequalSequenceNumbers() {
+ TrackingStruct tracking = new TrackingStruct(Tracking.schema());
+ tracking.set(0, EntryStatus.ADDED.id());
+
+ TrackingStruct manifestTracking = new TrackingStruct(Tracking.schema());
+ manifestTracking.set(0, EntryStatus.ADDED.id());
+ manifestTracking.set(1, 100L);
+ manifestTracking.set(2, 50L);
+ manifestTracking.set(3, 60L);
+
+ assertThatThrownBy(() -> tracking.inheritFrom(manifestTracking))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Manifest data and file sequence numbers must be equal, got 50 and 60");
+ }
+
+ @Test
+ void testNoDefaultingWithoutInheritance() {
+ TrackingStruct tracking = TrackingStruct.builder().status(EntryStatus.ADDED).build();
+
+ // no inheritance, nulls stay null
+ assertThat(tracking.snapshotId()).isNull();
+ assertThat(tracking.dataSequenceNumber()).isNull();
+ assertThat(tracking.fileSequenceNumber()).isNull();
+ }
+
+ private static Tracking createManifestTracking(long snapshotId, long sequenceNumber) {
+ return TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(snapshotId)
+ .dataSequenceNumber(sequenceNumber)
+ .fileSequenceNumber(sequenceNumber)
+ .build();
+ }
+
+ @Test
+ void testBuilderValidation() {
+ assertThatThrownBy(() -> TrackingStruct.builder().build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid status: null");
+ }
+
+ @Test
+ void testProjectedStructLike() {
+ // project only snapshot_id (field ID 1) and first_row_id (field ID 142)
+ Types.StructType projection = Types.StructType.of(Tracking.SNAPSHOT_ID, Tracking.FIRST_ROW_ID);
+
+ TrackingStruct tracking = new TrackingStruct(projection);
+ assertThat(tracking.size()).isEqualTo(2);
+
+ // projected position 0 maps to internal position 1 (snapshot_id)
+ // projected position 1 maps to internal position 5 (first_row_id)
+ tracking.set(0, 42L);
+ tracking.set(1, 1000L);
+
+ assertThat(tracking.snapshotId()).isEqualTo(42L);
+ assertThat(tracking.firstRowId()).isEqualTo(1000L);
+ assertThat(tracking.get(0, Long.class)).isEqualTo(42L);
+ assertThat(tracking.get(1, Long.class)).isEqualTo(1000L);
+ }
+
+ @Test
+ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(42L)
+ .dataSequenceNumber(10L)
+ .deletedPositions(new byte[] {1, 2})
+ .build();
+
+ TrackingStruct deserialized = TestHelpers.roundTripSerialize(tracking);
+
+ assertThat(deserialized.status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(deserialized.snapshotId()).isEqualTo(42L);
+ assertThat(deserialized.dataSequenceNumber()).isEqualTo(10L);
+ assertThat(deserialized.deletedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2}));
+ }
+
+ @Test
+ void testKryoSerializationRoundTrip() throws IOException {
+ TrackingStruct tracking =
+ TrackingStruct.builder()
+ .status(EntryStatus.ADDED)
+ .snapshotId(42L)
+ .dataSequenceNumber(10L)
+ .deletedPositions(new byte[] {1, 2})
+ .build();
+
+ TrackingStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(tracking);
+
+ assertThat(deserialized.status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(deserialized.snapshotId()).isEqualTo(42L);
+ assertThat(deserialized.dataSequenceNumber()).isEqualTo(10L);
+ assertThat(deserialized.deletedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2}));
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/avro/AvroTestHelpers.java b/core/src/test/java/org/apache/iceberg/avro/AvroTestHelpers.java
index 0a1cf43f4fb5..fd73706ce082 100644
--- a/core/src/test/java/org/apache/iceberg/avro/AvroTestHelpers.java
+++ b/core/src/test/java/org/apache/iceberg/avro/AvroTestHelpers.java
@@ -177,4 +177,12 @@ public static String readAvroCodec(File file) throws IOException {
return reader.getMetaString("avro.codec");
}
}
+
+ public static boolean hasIds(Schema schema) {
+ return AvroSchemaUtil.hasIds(schema);
+ }
+
+ public static Schema removeIds(org.apache.iceberg.Schema schema) {
+ return RemoveIds.removeIds(schema);
+ }
}
diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java
index 833b2fb0b46f..8997cf15a08c 100644
--- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java
+++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java
@@ -72,6 +72,8 @@
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.metrics.CommitReport;
import org.apache.iceberg.metrics.MetricsReport;
import org.apache.iceberg.metrics.MetricsReporter;
@@ -429,6 +431,44 @@ public void testDropNonEmptyNamespace() {
assertThat(catalog.namespaceExists(NS)).as("Namespace should not exist").isFalse();
}
+ @Test
+ public void testDropNamespaceWithNestedNamespace() {
+ assumeThat(supportsNestedNamespaces())
+ .as("Only valid when the catalog supports nested namespaces")
+ .isTrue();
+
+ C catalog = catalog();
+
+ Namespace parent = Namespace.of("parent");
+ Namespace nested = Namespace.of("parent", "child");
+
+ assertThat(catalog.namespaceExists(parent)).as("Parent namespace should not exist").isFalse();
+ assertThat(catalog.namespaceExists(nested)).as("Nested namespace should not exist").isFalse();
+
+ catalog.createNamespace(parent);
+ catalog.createNamespace(nested);
+
+ assertThat(catalog.namespaceExists(parent)).as("Parent namespace should exist").isTrue();
+ assertThat(catalog.namespaceExists(nested)).as("Nested namespace should exist").isTrue();
+
+ assertThatThrownBy(() -> catalog.dropNamespace(parent))
+ .isInstanceOf(NamespaceNotEmptyException.class)
+ .hasMessageContaining("is not empty");
+
+ assertThat(catalog.namespaceExists(parent)).as("Parent namespace should still exist").isTrue();
+ assertThat(catalog.namespaceExists(nested)).as("Nested namespace should still exist").isTrue();
+
+ assertThat(catalog.dropNamespace(nested))
+ .as("Dropping an existing nested namespace should return true")
+ .isTrue();
+ assertThat(catalog.namespaceExists(nested)).as("Nested namespace should not exist").isFalse();
+
+ assertThat(catalog.dropNamespace(parent))
+ .as("Dropping an existing namespace should return true")
+ .isTrue();
+ assertThat(catalog.namespaceExists(parent)).as("Parent namespace should not exist").isFalse();
+ }
+
@Test
public void testListNamespaces() {
C catalog = catalog();
@@ -1025,6 +1065,86 @@ public void testRenameTable() {
assertEmpty("Should not contain table after drop", catalog, NS);
}
+ @Test
+ public void createTableInUniqueLocation() {
+ Map additionalProperties =
+ ImmutableMap.of(CatalogProperties.UNIQUE_TABLE_LOCATION, "true");
+ C catalog = initCatalog("uniq_path_catalog", additionalProperties);
+
+ if (requiresNamespaceCreate()) {
+ catalog.createNamespace(NS);
+ }
+
+ catalog.createTable(TABLE, SCHEMA, PartitionSpec.unpartitioned());
+ catalog.renameTable(TABLE, RENAMED_TABLE);
+ catalog.createTable(TABLE, SCHEMA, PartitionSpec.unpartitioned());
+
+ Table table = catalog.loadTable(TABLE);
+ Table renamedTable = catalog.loadTable(RENAMED_TABLE);
+
+ assertThat(table.location())
+ .as("Tables %s and %s have different location", TABLE, RENAMED_TABLE)
+ .isNotEqualTo(renamedTable.location());
+ }
+
+ @Test
+ public void dropAfterRenameDoesntCorruptTable() throws IOException {
+ C catalog = catalog();
+
+ if (requiresNamespaceCreate()) {
+ catalog.createNamespace(TABLE.namespace());
+ }
+
+ PartitionSpec spec = PartitionSpec.unpartitioned();
+
+ Table initialTable = catalog.createTable(TABLE, SCHEMA, spec);
+ String initialFilePath = initialTable.locationProvider().newDataLocation("data-a.parquet");
+ DataFile dataFile =
+ DataFiles.builder(spec)
+ .withPath(initialFilePath)
+ .withFileSizeInBytes(10)
+ .withRecordCount(2)
+ .build();
+ initialTable.io().newOutputFile(initialFilePath).create().close();
+ initialTable.newAppend().appendFile(dataFile).commit();
+
+ catalog.renameTable(TABLE, RENAMED_TABLE);
+
+ Table newTable = catalog.createTable(TABLE, SCHEMA, spec);
+ String newFilePath = newTable.locationProvider().newDataLocation("data-b.parquet");
+ DataFile anotherFile =
+ DataFiles.builder(spec)
+ .withPath(newFilePath)
+ .withFileSizeInBytes(10)
+ .withRecordCount(2)
+ .build();
+ newTable.io().newOutputFile(newFilePath).create().close();
+ newTable.newAppend().appendFile(anotherFile).commit();
+
+ catalog.dropTable(RENAMED_TABLE, true);
+
+ assertThat(catalog.tableExists(RENAMED_TABLE))
+ .as("After PURGE, %s must not exist", RENAMED_TABLE)
+ .isFalse();
+ assertThat(catalog.tableExists(TABLE))
+ .as(
+ "After dropping the renamed table with PURGE, the recreated table with the original name (%s) must exist",
+ TABLE)
+ .isTrue();
+
+ Table table = catalog.loadTable(TABLE);
+ FileIO io = table.io();
+ try (CloseableIterable tasks = table.newScan().planFiles()) {
+ tasks.forEach(
+ task -> {
+ InputFile file = io.newInputFile(task.file().location());
+ assertThat(file.exists())
+ .as("Table %s should remain unaffected by dropping %s", TABLE, RENAMED_TABLE)
+ .isTrue();
+ });
+ }
+ }
+
@Test
public void testRenameTableMissingSourceTable() {
C catalog = catalog();
diff --git a/data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java b/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java
similarity index 100%
rename from data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java
rename to core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java
diff --git a/core/src/test/java/org/apache/iceberg/deletes/TestRoaringPositionBitmap.java b/core/src/test/java/org/apache/iceberg/deletes/TestRoaringPositionBitmap.java
index 2daf0382973b..68b73ed0a218 100644
--- a/core/src/test/java/org/apache/iceberg/deletes/TestRoaringPositionBitmap.java
+++ b/core/src/test/java/org/apache/iceberg/deletes/TestRoaringPositionBitmap.java
@@ -148,9 +148,116 @@ public void testAddRangeAcrossKeys() {
@TestTemplate
public void testAddEmptyRange() {
+ RoaringPositionBitmap equalRange = new RoaringPositionBitmap();
+ equalRange.setRange(10, 10);
+ assertThat(equalRange.isEmpty()).isTrue();
+ assertThat(equalRange.cardinality()).isEqualTo(0);
+ assertThat(equalRange.contains(10)).isFalse();
+ }
+
+ @TestTemplate
+ public void testSetRangeReversedThrows() {
+ RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
+ assertThatThrownBy(() -> bitmap.setRange(100, 50))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Start position must not exceed end position");
+ }
+
+ @TestTemplate
+ public void testAddRangeLargeContiguous() {
+ RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
+
+ long start = 500L;
+ long end = 200_500L;
+ bitmap.setRange(start, end);
+
+ assertThat(bitmap.cardinality()).isEqualTo(200_000L);
+ assertThat(bitmap.contains(start)).isTrue();
+ assertThat(bitmap.contains(end - 1)).isTrue();
+ assertThat(bitmap.contains(start - 1)).isFalse();
+ assertThat(bitmap.contains(end)).isFalse();
+ }
+
+ @TestTemplate
+ public void testAddRangeSpanningThreeKeys() {
+ RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
+
+ long start = ((long) 0 << 32) | 0xFFFFFFF0L;
+ long end = ((long) 2 << 32) | 0x10L;
+ bitmap.setRange(start, end);
+
+ assertThat(bitmap.contains(start)).isTrue();
+ assertThat(bitmap.contains(end - 1)).isTrue();
+ assertThat(bitmap.contains(start - 1)).isFalse();
+ assertThat(bitmap.contains(end)).isFalse();
+
+ // key 1 should be fully covered
+ assertThat(bitmap.contains((long) 1 << 32)).isTrue();
+ assertThat(bitmap.contains(((long) 1 << 32) | 0xFFFFFFFFL)).isTrue();
+
+ long expectedCardinality = end - start;
+ assertThat(bitmap.cardinality()).isEqualTo(expectedCardinality);
+ }
+
+ @TestTemplate
+ public void testAddRangeSinglePosition() {
+ RoaringPositionBitmap rangeBitmap = new RoaringPositionBitmap();
+ rangeBitmap.setRange(42, 43);
+
+ RoaringPositionBitmap setBitmap = new RoaringPositionBitmap();
+ setBitmap.set(42);
+
+ assertThat(rangeBitmap.cardinality()).isEqualTo(setBitmap.cardinality());
+ assertThat(rangeBitmap.contains(42)).isTrue();
+ assertThat(rangeBitmap.contains(41)).isFalse();
+ assertThat(rangeBitmap.contains(43)).isFalse();
+ }
+
+ @TestTemplate
+ public void testAddRangeAtKeyBoundary() {
+ RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
+
+ bitmap.setRange(0L, 1L << 32);
+
+ assertThat(bitmap.cardinality()).isEqualTo(1L << 32);
+ assertThat(bitmap.contains(0L)).isTrue();
+ assertThat(bitmap.contains((1L << 32) - 1)).isTrue();
+ assertThat(bitmap.contains(1L << 32)).isFalse();
+ assertThat(bitmap.allocatedBitmapCount()).isEqualTo(1);
+ }
+
+ @TestTemplate
+ public void testAddRangeSameKeyForEachExact() {
RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
- bitmap.setRange(10, 10);
- assertThat(bitmap.isEmpty()).isTrue();
+
+ long start = 1000L;
+ long end = 1200L;
+ bitmap.setRange(start, end);
+
+ assertThat(bitmap.cardinality()).isEqualTo(end - start);
+ assertThat(bitmap.contains(start - 1)).isFalse();
+ assertThat(bitmap.contains(end)).isFalse();
+
+ for (long pos = start; pos < end; pos++) {
+ assertThat(bitmap.contains(pos)).isTrue();
+ }
+ }
+
+ @TestTemplate
+ public void testAddRangeCrossKeyForEachExact() {
+ RoaringPositionBitmap bitmap = new RoaringPositionBitmap();
+
+ long start = ((long) 1 << 32) - 100L;
+ long end = ((long) 1 << 32) + 100L;
+ bitmap.setRange(start, end);
+
+ assertThat(bitmap.cardinality()).isEqualTo(end - start);
+ assertThat(bitmap.contains(start - 1)).isFalse();
+ assertThat(bitmap.contains(end)).isFalse();
+
+ for (long pos = start; pos < end; pos++) {
+ assertThat(bitmap.contains(pos)).isTrue();
+ }
}
@TestTemplate
@@ -357,6 +464,18 @@ public void testUnsupportedPositions() {
.hasMessageContaining(
"Bitmap supports positions that are >= 0 and <= %s",
RoaringPositionBitmap.MAX_POSITION);
+
+ assertThatThrownBy(() -> bitmap.setRange(-1L, 1L))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Bitmap supports positions that are >= 0 and <= %s",
+ RoaringPositionBitmap.MAX_POSITION);
+
+ assertThatThrownBy(() -> bitmap.setRange(0L, RoaringPositionBitmap.MAX_POSITION + 2L))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Bitmap supports positions that are >= 0 and <= %s",
+ RoaringPositionBitmap.MAX_POSITION);
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java
index c2c683e7d882..827450d4a398 100644
--- a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java
@@ -82,6 +82,11 @@ protected boolean supportsEmptyNamespace() {
return true;
}
+ @Override
+ protected boolean supportsNestedNamespaces() {
+ return true;
+ }
+
@Test
@Override
public void testLoadTableWithMissingMetadataFile(@TempDir Path tempDir) throws IOException {
diff --git a/core/src/test/java/org/apache/iceberg/io/TestBufferedFileAppender.java b/core/src/test/java/org/apache/iceberg/io/TestBufferedFileAppender.java
new file mode 100644
index 000000000000..9bbc0f9f8c71
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/io/TestBufferedFileAppender.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.io;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.avro.Avro;
+import org.apache.iceberg.avro.AvroIterable;
+import org.apache.iceberg.data.DataTestHelpers;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.data.avro.DataWriter;
+import org.apache.iceberg.data.avro.PlannedDataReader;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.inmemory.InMemoryOutputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class TestBufferedFileAppender {
+
+ private static final Schema SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()));
+
+ private InMemoryOutputFile outputFile;
+ private GenericRecord record;
+
+ @BeforeEach
+ public void before() {
+ this.outputFile = new InMemoryOutputFile();
+ this.record = GenericRecord.create(SCHEMA);
+ }
+
+ private Function, FileAppender> avroFactory(OutputFile out) {
+ return bufferedRows -> {
+ try {
+ return Avro.write(out)
+ .createWriterFunc(DataWriter::create)
+ .schema(SCHEMA)
+ .overwrite()
+ .build();
+ } catch (IOException e) {
+ throw new RuntimeIOException(e);
+ }
+ };
+ }
+
+ private BufferedFileAppender createAppender(int bufferSize) {
+ return new BufferedFileAppender<>(bufferSize, avroFactory(outputFile), Record::copy);
+ }
+
+ private Record createRecord(long id, String data) {
+ return record.copy(ImmutableMap.of("id", id, "data", data));
+ }
+
+ private List readBack() throws IOException {
+ try (AvroIterable reader =
+ Avro.read(outputFile.toInputFile())
+ .project(SCHEMA)
+ .createResolvingReader(PlannedDataReader::create)
+ .build()) {
+ return Lists.newArrayList(reader);
+ }
+ }
+
+ @Test
+ public void testBufferFlushesOnThreshold() throws IOException {
+ BufferedFileAppender appender = createAppender(3);
+
+ appender.add(createRecord(1L, "a"));
+ appender.add(createRecord(2L, "b"));
+
+ // delegate not yet created, length should be 0
+ assertThat(appender.length()).isEqualTo(0L);
+
+ appender.add(createRecord(3L, "c"));
+
+ // delegate created after 3rd row, length should be > 0
+ assertThat(appender.length()).isGreaterThan(0L);
+
+ appender.add(createRecord(4L, "d"));
+ appender.add(createRecord(5L, "e"));
+ appender.close();
+
+ List expected =
+ Lists.newArrayList(
+ createRecord(1L, "a"),
+ createRecord(2L, "b"),
+ createRecord(3L, "c"),
+ createRecord(4L, "d"),
+ createRecord(5L, "e"));
+ DataTestHelpers.assertEquals(SCHEMA.asStruct(), expected, readBack());
+ }
+
+ @Test
+ public void testCloseWithPartialBuffer() throws IOException {
+ BufferedFileAppender appender = createAppender(10);
+
+ appender.add(createRecord(1L, "a"));
+ appender.add(createRecord(2L, "b"));
+ appender.add(createRecord(3L, "c"));
+
+ // buffer not full yet
+ assertThat(appender.length()).isEqualTo(0L);
+
+ // close flushes partial buffer through factory
+ appender.close();
+
+ List expected =
+ Lists.newArrayList(createRecord(1L, "a"), createRecord(2L, "b"), createRecord(3L, "c"));
+ DataTestHelpers.assertEquals(SCHEMA.asStruct(), expected, readBack());
+ }
+
+ @Test
+ public void testCopyFuncIsApplied() throws IOException {
+ BufferedFileAppender appender = createAppender(3);
+
+ // use a single mutable record, relying on copyFunc to snapshot it
+ record.set(0, 1L);
+ record.set(1, "first");
+ appender.add(record);
+
+ record.set(0, 2L);
+ record.set(1, "second");
+ appender.add(record);
+
+ record.set(0, 3L);
+ record.set(1, "third");
+ appender.add(record);
+
+ appender.close();
+
+ List expected =
+ Lists.newArrayList(
+ createRecord(1L, "first"), createRecord(2L, "second"), createRecord(3L, "third"));
+ DataTestHelpers.assertEquals(SCHEMA.asStruct(), expected, readBack());
+ }
+
+ @Test
+ public void testMetricsAfterClose() throws IOException {
+ BufferedFileAppender appender = createAppender(2);
+
+ appender.add(createRecord(1L, "a"));
+ appender.add(createRecord(2L, "b"));
+ appender.add(createRecord(3L, "c"));
+ appender.close();
+
+ assertThat(appender.metrics()).isNotNull();
+ assertThat(appender.metrics().recordCount()).isEqualTo(3L);
+ assertThat(appender.length()).isGreaterThan(0L);
+ }
+
+ @Test
+ public void testMetricsBeforeCloseThrows() throws IOException {
+ try (BufferedFileAppender appender = createAppender(10)) {
+ assertThatThrownBy(appender::metrics)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Cannot return metrics for unclosed appender");
+ }
+ }
+
+ @Test
+ public void testAddAfterCloseThrows() throws IOException {
+ try (BufferedFileAppender appender = createAppender(10)) {
+ appender.add(createRecord(1L, "a"));
+ appender.close();
+
+ assertThatThrownBy(() -> appender.add(createRecord(2L, "b")))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Cannot add to a closed appender");
+ }
+ }
+
+ @Test
+ public void testAddAllSpanningBuffer() throws IOException {
+ BufferedFileAppender appender = createAppender(2);
+
+ List records =
+ Lists.newArrayList(
+ createRecord(1L, "a"),
+ createRecord(2L, "b"),
+ createRecord(3L, "c"),
+ createRecord(4L, "d"));
+
+ appender.addAll(records);
+ appender.close();
+
+ DataTestHelpers.assertEquals(SCHEMA.asStruct(), records, readBack());
+ }
+
+ @Test
+ public void testCloseWithNoData() throws IOException {
+ BufferedFileAppender appender = createAppender(10);
+ // close immediately with no data written
+ appender.close();
+ // delegate was never created
+ assertThat(appender.length()).isEqualTo(0L);
+ assertThat(appender.metrics()).isNotNull();
+ assertThat(appender.metrics().recordCount()).isEqualTo(0L);
+ assertThat(appender.splitOffsets()).isNull();
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
index 310d918849f3..ff0af5c56306 100644
--- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
@@ -852,11 +852,11 @@ public void testDropNamespace() {
assertThatThrownBy(() -> catalog.dropNamespace(tbl2.namespace()))
.isInstanceOf(NamespaceNotEmptyException.class)
- .hasMessage("Namespace db.ns1 is not empty. Contains 1 table(s).");
+ .hasMessage("Namespace db.ns1 is not empty. Contains 1 child namespace(s).");
assertThatThrownBy(() -> catalog.dropNamespace(tbl4.namespace()))
.isInstanceOf(NamespaceNotEmptyException.class)
- .hasMessage("Namespace db is not empty. Contains 1 table(s).");
+ .hasMessage("Namespace db is not empty. Contains 2 child namespace(s).");
}
@Test
diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
index 8ba5daef3f9b..8c6dc52b1575 100644
--- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
+++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
@@ -746,7 +746,7 @@ private static SnapshotMode snapshotModeFromQueryParams(Map