Skip to content

Encryption for REST catalog #13225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
smaheshwar-pltr wants to merge 18 commits into
base: main
Choose a base branch
from
Open

Encryption for REST catalog #13225

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
af672ed
Encryption for REST catalog
smaheshwar-pltr Dec 5, 2025
7ad846d
Nit improvements
smaheshwar-pltr Dec 5, 2025
9ed8d8e
Suppress test method length
smaheshwar-pltr Dec 5, 2025
2d18a6a
Merge branch 'main' into rest-encrypt-on-main-cherry-pick
smaheshwar-pltr Dec 9, 2025
745a959
Spotless
smaheshwar-pltr Dec 9, 2025
1106a26
Merge branch 'main' into rest-encrypt-on-main-cherry-pick
smaheshwar-pltr Jan 21, 2026
8ac5228
Fix conflicts and update test
smaheshwar-pltr Jan 21, 2026
73e0980
Undo 4.0 changes
smaheshwar-pltr Jan 21, 2026
a774a9c
Fix checkstyle
smaheshwar-pltr Jan 21, 2026
86abfab
empty
smaheshwar-pltr Jan 21, 2026
1e42400
Nit: Simplify adding encryption keys
smaheshwar-pltr Jan 21, 2026
3c055ce
Respect `CatalogProperties.ENCRYPTION_KMS_TYPE`
smaheshwar-pltr Mar 12, 2026
b9e5822
Merge
smaheshwar-pltr Mar 12, 2026
cd04290
Fix merge
smaheshwar-pltr Mar 12, 2026
ee27d5d
Nits + spotless
smaheshwar-pltr Mar 12, 2026
2092454
Add TODO for encryption <> scan planning
smaheshwar-pltr May 1, 2026
cfb7345
Fixes
smaheshwar-pltr May 2, 2026
0af4ef4
Merge remote-tracking branch 'origin/main' into rest-encrypt-on-main-…
smaheshwar-pltr May 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableCommit;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.encryption.EncryptionUtil;
import org.apache.iceberg.encryption.KeyManagementClient;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
Expand Down Expand Up @@ -161,6 +163,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog
private CloseableGroup closeables = null;
private Set<Endpoint> endpoints;
private Supplier<Map<String, String>> mutationHeaders = Map::of;
private KeyManagementClient keyManagementClient = null;

public RESTSessionCatalog() {
this(
Expand Down Expand Up @@ -263,6 +266,12 @@ public void initialize(String name, Map<String, String> unresolved) {
mergedProps,
RESTCatalogProperties.METRICS_REPORTING_ENABLED,
RESTCatalogProperties.METRICS_REPORTING_ENABLED_DEFAULT);

if (mergedProps.containsKey(CatalogProperties.ENCRYPTION_KMS_IMPL)) {
this.keyManagementClient = EncryptionUtil.createKmsClient(mergedProps);
this.closeables.addCloseable(this.keyManagementClient);
}

super.initialize(name, mergedProps);
}

Expand Down Expand Up @@ -464,6 +473,7 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) {
Map::of,
mutationHeaders,
tableFileIO(context, tableConf, response.credentials()),
keyManagementClient,
tableMetadata,
endpoints);

Expand Down Expand Up @@ -544,6 +554,7 @@ public Table registerTable(
Map::of,
mutationHeaders,
tableFileIO(context, tableConf, response.credentials()),
keyManagementClient,
response.tableMetadata(),
endpoints);

Expand Down Expand Up @@ -808,6 +819,7 @@ public Table create() {
Map::of,
mutationHeaders,
tableFileIO(context, tableConf, response.credentials()),
keyManagementClient,
response.tableMetadata(),
endpoints);

Expand Down Expand Up @@ -836,6 +848,7 @@ public Transaction createTransaction() {
Map::of,
mutationHeaders,
tableFileIO(context, tableConf, response.credentials()),
keyManagementClient,
RESTTableOperations.UpdateType.CREATE,
createChanges(meta),
meta,
Expand Down Expand Up @@ -900,6 +913,7 @@ public Transaction replaceTransaction() {
Map::of,
mutationHeaders,
tableFileIO(context, tableConf, response.credentials()),
keyManagementClient,
RESTTableOperations.UpdateType.REPLACE,
changes.build(),
base,
Expand Down Expand Up @@ -1040,6 +1054,7 @@ private FileIO tableFileIO(
* @param mutationHeaderSupplier a supplier for additional HTTP headers to include in mutation
* requests (POST/DELETE)
* @param fileIO the FileIO implementation for reading and writing table metadata and data files
* @param kmsClient the {@link KeyManagementClient} for encrypted tables
Copy link
Copy Markdown
Contributor Author

@smaheshwar-pltr smaheshwar-pltr Jan 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pointing out that I'm changing the signature of this method. #14740 did the same recently, and I think #14465 that hasn't released yet so there shouldn't be problems with this

* @param current the current table metadata
* @param supportedEndpoints the set of supported REST endpoints
* @return a new RESTTableOperations instance
Expand All @@ -1050,10 +1065,18 @@ protected RESTTableOperations newTableOps(
Supplier<Map<String, String>> readHeaders,
Supplier<Map<String, String>> mutationHeaderSupplier,
FileIO fileIO,
KeyManagementClient kmsClient,
TableMetadata current,
Set<Endpoint> supportedEndpoints) {
return new RESTTableOperations(
restClient, path, readHeaders, mutationHeaderSupplier, fileIO, current, supportedEndpoints);
restClient,
path,
readHeaders,
mutationHeaderSupplier,
fileIO,
kmsClient,
current,
supportedEndpoints);
}

/**
Expand All @@ -1070,6 +1093,7 @@ protected RESTTableOperations newTableOps(
* @param mutationHeaderSupplier a supplier for additional HTTP headers to include in mutation
* requests (POST/DELETE)
* @param fileIO the FileIO implementation for reading and writing table metadata and data files
* @param kmsClient the {@link KeyManagementClient} for encrypted tables
* @param updateType the {@link RESTTableOperations.UpdateType} being performed
* @param createChanges the list of metadata updates to apply during table creation or replacement
* @param current the current table metadata (may be null for CREATE operations)
Expand All @@ -1082,6 +1106,7 @@ protected RESTTableOperations newTableOps(
Supplier<Map<String, String>> readHeaders,
Supplier<Map<String, String>> mutationHeaderSupplier,
FileIO fileIO,
KeyManagementClient kmsClient,
RESTTableOperations.UpdateType updateType,
List<MetadataUpdate> createChanges,
TableMetadata current,
Expand All @@ -1092,6 +1117,7 @@ protected RESTTableOperations newTableOps(
readHeaders,
mutationHeaderSupplier,
fileIO,
kmsClient,
updateType,
createChanges,
current,
Expand Down
Loading