diff --git a/build.gradle b/build.gradle index 261dfabf0412..2ed6c0fe2fb3 100644 --- a/build.gradle +++ b/build.gradle @@ -802,6 +802,30 @@ project(':iceberg-gcp') { } } +project(':iceberg-hashicorp') { + apply plugin: 'com.gradleup.shadow' + + build.dependsOn shadowJar + + test { + useJUnitPlatform() + } + + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + implementation project(':iceberg-core') + implementation libs.httpcomponents.httpclient5 + implementation libs.jackson.databind + + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation libs.esotericsoftware.kryo + testImplementation libs.jackson.databind + testImplementation libs.testcontainers + testImplementation libs.testcontainers.junit.jupiter + testImplementation libs.testcontainers.vault + } +} + project(':iceberg-hive-metastore') { test { useJUnitPlatform() diff --git a/core/src/main/java/org/apache/iceberg/CatalogProperties.java b/core/src/main/java/org/apache/iceberg/CatalogProperties.java index 6b85ccbc87bc..ae1376f2bccc 100644 --- a/core/src/main/java/org/apache/iceberg/CatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/CatalogProperties.java @@ -174,6 +174,7 @@ private CatalogProperties() {} public static final String ENCRYPTION_KMS_TYPE_AWS = "aws"; public static final String ENCRYPTION_KMS_TYPE_AZURE = "azure"; public static final String ENCRYPTION_KMS_TYPE_GCP = "gcp"; + public static final String ENCRYPTION_KMS_TYPE_HASHICORP = "hashicorp"; public static final String ENCRYPTION_KMS_IMPL = "encryption.kms-impl"; public static final String ENCRYPTION_KMS_IMPL_AWS = @@ -182,4 +183,6 @@ private CatalogProperties() {} "org.apache.iceberg.azure.keymanagement.AzureKeyManagementClient"; public static final String ENCRYPTION_KMS_IMPL_GCP = "org.apache.iceberg.gcp.GcpKeyManagementClient"; + public static final String ENCRYPTION_KMS_IMPL_HASHICORP = + "org.apache.iceberg.hashicorp.VaultKeyManagementClient"; } diff --git a/core/src/main/java/org/apache/iceberg/encryption/EncryptionUtil.java b/core/src/main/java/org/apache/iceberg/encryption/EncryptionUtil.java index 382d244883d6..4be58f78e315 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/EncryptionUtil.java +++ b/core/src/main/java/org/apache/iceberg/encryption/EncryptionUtil.java @@ -63,6 +63,8 @@ public static KeyManagementClient createKmsClient(Map catalogPro CatalogProperties.ENCRYPTION_KMS_IMPL_AZURE; case CatalogProperties.ENCRYPTION_KMS_TYPE_GCP -> CatalogProperties.ENCRYPTION_KMS_IMPL_GCP; + case CatalogProperties.ENCRYPTION_KMS_TYPE_HASHICORP -> + CatalogProperties.ENCRYPTION_KMS_IMPL_HASHICORP; default -> throw new IllegalStateException("Unsupported KMS type: " + kmsType); }; } diff --git a/docs/docs/encryption.md b/docs/docs/encryption.md index cbdce85e760e..64a17a9124ab 100644 --- a/docs/docs/encryption.md +++ b/docs/docs/encryption.md @@ -28,7 +28,7 @@ Currently, encryption is supported in the Hive and REST catalogs for tables with Two parameters are required to activate encryption of a table: -1. Catalog property that specifies the KMS ("key management service"). It can be either `encryption.kms-type` for pre-defined KMS clients (`aws`, `azure` or `gcp`) or `encryption.kms-impl` with the client class path for custom KMS clients. +1. Catalog property that specifies the KMS ("key management service"). It can be either `encryption.kms-type` for pre-defined KMS clients (`aws`, `azure`, `gcp` or `hashicorp`) or `encryption.kms-impl` with the client class path for custom KMS clients. 2. Table property `encryption.key-id`, that specifies the ID of a master key used to encrypt and decrypt the table. Master keys are stored and managed in the KMS. For more details on table encryption, see the "Appendix: Internals Overview" [subsection](#appendix-internals-overview). diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8b24d4727436..474e3cd8779b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -229,5 +229,6 @@ sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } testcontainers-junit-jupiter = { module = "org.testcontainers:testcontainers-junit-jupiter", version.ref = "testcontainers" } testcontainers-minio = { module = "org.testcontainers:testcontainers-minio", version.ref = "testcontainers" } +testcontainers-vault = { module = "org.testcontainers:testcontainers-vault", version.ref = "testcontainers" } tez08-dag = { module = "org.apache.tez:tez-dag", version.ref = "tez08" } tez08-mapreduce = { module = "org.apache.tez:tez-mapreduce", version.ref = "tez08" } diff --git a/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultClient.java b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultClient.java new file mode 100644 index 000000000000..207bc3e438ac --- /dev/null +++ b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultClient.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.hashicorp; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.OptionalLong; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +/** + * HTTP client for interacting with HashiCorp Vault REST API. + * + * @see HashiCorp Vault HTTP API + */ +class VaultClient implements Closeable { + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String VAULT_TOKEN_HEADER = "X-Vault-Token"; + + private final String address; + private final String transitMount; + private final String appRolePath; + + private transient volatile CloseableHttpClient httpClient; + + VaultClient(String address, String transitMount, String appRolePath) { + this.address = address; + this.transitMount = transitMount; + this.appRolePath = appRolePath; + } + + AuthResult authenticate(String roleId, String secretId) { + ObjectNode requestBody = MAPPER.createObjectNode(); + requestBody.put("role_id", roleId); + requestBody.put("secret_id", secretId); + + JsonNode response = post("/v1/auth/" + appRolePath + "/login", null, requestBody); + JsonNode authNode = response.get("auth"); + if (authNode == null) { + throw new RuntimeException("Failed to authenticate: no auth section in response"); + } + + String clientToken = authNode.get("client_token").asText(); + OptionalLong leaseDuration = + authNode.has("lease_duration") + ? OptionalLong.of(authNode.get("lease_duration").asLong()) + : OptionalLong.empty(); + return new AuthResult(clientToken, leaseDuration); + } + + String encrypt(String vaultToken, String wrappingKeyId, String plaintext) { + ObjectNode requestBody = MAPPER.createObjectNode(); + requestBody.put("plaintext", plaintext); + + JsonNode response = + post("/v1/" + transitMount + "/encrypt/" + wrappingKeyId, vaultToken, requestBody); + + JsonNode dataNode = response.get("data"); + if (dataNode == null || !dataNode.has("ciphertext")) { + throw new RuntimeException("Failed to wrap key: no ciphertext returned"); + } + + return dataNode.get("ciphertext").asText(); + } + + String decrypt(String vaultToken, String wrappingKeyId, String ciphertext) { + ObjectNode requestBody = MAPPER.createObjectNode(); + requestBody.put("ciphertext", ciphertext); + + JsonNode response = + post("/v1/" + transitMount + "/decrypt/" + wrappingKeyId, vaultToken, requestBody); + + JsonNode dataNode = response.get("data"); + if (dataNode == null || !dataNode.has("plaintext")) { + throw new RuntimeException("Failed to unwrap key: no plaintext returned"); + } + + return dataNode.get("plaintext").asText(); + } + + DataKey generateKey(String vaultToken, String wrappingKeyId) { + ObjectNode requestBody = MAPPER.createObjectNode(); + + JsonNode response = + post( + "/v1/" + transitMount + "/datakey/plaintext/" + wrappingKeyId, vaultToken, requestBody); + + JsonNode dataNode = response.get("data"); + if (dataNode == null || !dataNode.has("plaintext") || !dataNode.has("ciphertext")) { + throw new RuntimeException("Failed to generate key: missing plaintext or ciphertext"); + } + + String plaintext = dataNode.get("plaintext").asText(); + String ciphertext = dataNode.get("ciphertext").asText(); + return new DataKey(plaintext, ciphertext); + } + + private JsonNode post(String path, String token, ObjectNode requestBody) { + HttpPost request = new HttpPost(address + path); + if (token != null) { + request.setHeader(VAULT_TOKEN_HEADER, token); + } + + try { + request.setEntity( + new StringEntity(MAPPER.writeValueAsString(requestBody), ContentType.APPLICATION_JSON)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to serialize request body", e); + } + + try { + return httpClient().execute(request, HttpClientContext.create(), new VaultResponseHandler()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to execute Vault request to " + path, e); + } + } + + private static class VaultResponseHandler implements HttpClientResponseHandler { + @Override + public JsonNode handleResponse(ClassicHttpResponse response) { + int statusCode = response.getCode(); + Preconditions.checkState(statusCode == 200, "Status must be 200: %d", statusCode); + + try { + String responseBody = EntityUtils.toString(response.getEntity()); + return MAPPER.readTree(responseBody); + } catch (ParseException e) { + throw new RuntimeException("Failed to parse Vault error response", e); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read response", e); + } + } + } + + private CloseableHttpClient httpClient() { + if (httpClient == null) { + synchronized (this) { + if (httpClient == null) { + httpClient = HttpClients.createDefault(); + } + } + } + return httpClient; + } + + @Override + public void close() { + if (httpClient != null) { + try { + httpClient.close(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to close HTTP client", e); + } + } + } + + static class AuthResult { + private final String clientToken; + private final OptionalLong leaseDuration; + + AuthResult(String clientToken, OptionalLong leaseDuration) { + this.clientToken = clientToken; + this.leaseDuration = leaseDuration; + } + + public String clientToken() { + return clientToken; + } + + public OptionalLong leaseDuration() { + return leaseDuration; + } + } + + static class DataKey { + private final String plaintext; + private final String ciphertext; + + DataKey(String plaintext, String ciphertext) { + this.plaintext = plaintext; + this.ciphertext = ciphertext; + } + + public String plaintext() { + return plaintext; + } + + public String ciphertext() { + return ciphertext; + } + } +} diff --git a/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultKeyManagementClient.java b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultKeyManagementClient.java new file mode 100644 index 000000000000..1d304236014f --- /dev/null +++ b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultKeyManagementClient.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.hashicorp; + +import java.io.Closeable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; +import org.apache.iceberg.encryption.KeyManagementClient; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.util.SerializableMap; + +/** + * KMS client implementation using HashiCorp Vault Transit secrets engine with AppRole + * authentication. + */ +public class VaultKeyManagementClient implements KeyManagementClient, Closeable { + @SuppressWarnings("unused") + private SerializableMap properties; + + private String vaultAddress; + private String transitMount; + private String appRolePath; + private String appRoleId; + private String appSecretId; + private boolean rotateToken; + + private transient volatile String vaultToken; + private transient volatile long tokenExpiry; + private transient volatile VaultClient client; + + @Override + public void initialize(Map newProperties) { + this.properties = SerializableMap.copyOf(newProperties); + + vaultAddress = newProperties.get(VaultProperties.VAULT_ADDRESS_PROP); + Preconditions.checkArgument( + !Strings.isNullOrEmpty(vaultAddress), + "%s must be set in newProperties", + VaultProperties.VAULT_ADDRESS_PROP); + + transitMount = newProperties.getOrDefault(VaultProperties.VAULT_TRANSIT_MOUNT_PROP, "transit"); + appRolePath = newProperties.getOrDefault(VaultProperties.VAULT_APPROLE_PATH_PROP, "approle"); + rotateToken = + Boolean.parseBoolean( + newProperties.getOrDefault(VaultProperties.VAULT_ROTATE_TOKEN_PROP, "false")); + + String configuredVaultToken = newProperties.get(VaultProperties.VAULT_TOKEN_PROP); + appRoleId = newProperties.get(VaultProperties.VAULT_ROLE_ID_PROP); + appSecretId = + newProperties.getOrDefault( + VaultProperties.VAULT_SECRET_ID_PROP, + System.getenv(VaultProperties.VAULT_SECRET_ID_ENV_VAR)); + + boolean hasTokenAuth = !Strings.isNullOrEmpty(configuredVaultToken); + boolean hasAppRoleAuth = + !Strings.isNullOrEmpty(appRoleId) && !Strings.isNullOrEmpty(appSecretId); + + Preconditions.checkArgument( + hasTokenAuth || hasAppRoleAuth, + "Either token or both role id and secret id must be set in newProperties"); + + // Initialize vaultToken for direct token auth (don't mark as transient field) + if (hasTokenAuth) { + vaultToken = configuredVaultToken; + } + } + + private void authenticate() { + if (Strings.isNullOrEmpty(vaultToken)) { + authenticateWithAppRole(); + } + } + + private void authenticateWithAppRole() { + VaultClient.AuthResult result = client().authenticate(appRoleId, appSecretId); + vaultToken = result.clientToken(); + + if (rotateToken) { + long leaseDuration = result.leaseDuration().orElseThrow(); + tokenExpiry = System.currentTimeMillis() + (leaseDuration * 1000); + } + } + + @Override + public ByteBuffer wrapKey(ByteBuffer key, String wrappingKeyId) { + ensureValidToken(); + + byte[] keyBytes = new byte[key.remaining()]; + key.duplicate().get(keyBytes); + String plaintext = Base64.getEncoder().encodeToString(keyBytes); + + String ciphertext = client().encrypt(vaultToken, wrappingKeyId, plaintext); + return ByteBuffer.wrap(ciphertext.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public ByteBuffer unwrapKey(ByteBuffer wrappedKey, String wrappingKeyId) { + ensureValidToken(); + + byte[] ciphertextBytes = new byte[wrappedKey.remaining()]; + wrappedKey.duplicate().get(ciphertextBytes); + String ciphertext = new String(ciphertextBytes, StandardCharsets.UTF_8); + + String plaintext = client().decrypt(vaultToken, wrappingKeyId, ciphertext); + byte[] keyBytes = Base64.getDecoder().decode(plaintext); + return ByteBuffer.wrap(keyBytes); + } + + @Override + public boolean supportsKeyGeneration() { + return true; + } + + @Override + public KeyGenerationResult generateKey(String wrappingKeyId) { + ensureValidToken(); + + VaultClient.DataKey dataKey = client().generateKey(vaultToken, wrappingKeyId); + + byte[] keyBytes = Base64.getDecoder().decode(dataKey.plaintext()); + ByteBuffer key = ByteBuffer.wrap(keyBytes); + ByteBuffer wrappedKey = ByteBuffer.wrap(dataKey.ciphertext().getBytes(StandardCharsets.UTF_8)); + + return new KeyGenerationResult(key, wrappedKey); + } + + private void ensureValidToken() { + if (vaultToken == null) { + synchronized (this) { + if (vaultToken == null) { + // After deserialization, restore token from properties if using token auth + String configuredToken = properties.get(VaultProperties.VAULT_TOKEN_PROP); + if (!Strings.isNullOrEmpty(configuredToken)) { + vaultToken = configuredToken; + } else { + authenticate(); + } + } + } + } + + if (!rotateToken) { + return; + } + + if (System.currentTimeMillis() > (tokenExpiry - 300_000)) { + synchronized (this) { + if (System.currentTimeMillis() > (tokenExpiry - 300_000)) { + authenticateWithAppRole(); + } + } + } + } + + private VaultClient client() { + if (client == null) { + synchronized (this) { + if (client == null) { + client = new VaultClient(vaultAddress, transitMount, appRolePath); + } + } + } + return client; + } + + @Override + public void close() { + if (client != null) { + client.close(); + } + } +} diff --git a/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultProperties.java b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultProperties.java new file mode 100644 index 000000000000..c3927d607b47 --- /dev/null +++ b/hashicorp/src/main/java/org/apache/iceberg/hashicorp/VaultProperties.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.hashicorp; + +import java.io.Serializable; + +public final class VaultProperties implements Serializable { + /** Vault server address */ + public static final String VAULT_ADDRESS_PROP = "vault.address"; + + /** Direct token authentication */ + public static final String VAULT_TOKEN_PROP = "vault.token"; + + /** Transit secrets engine mount path */ + public static final String VAULT_TRANSIT_MOUNT_PROP = "vault.transit-mount"; + + /** AppRole authentication path */ + public static final String VAULT_APPROLE_PATH_PROP = "vault.approle-path"; + + /** Role ID for AppRole authentication */ + public static final String VAULT_ROLE_ID_PROP = "vault.role-id"; + + /** Secret ID for AppRole authentication */ + public static final String VAULT_SECRET_ID_PROP = "vault.secret-id"; + + /** Environment variable of Secret ID for AppRole authentication */ + public static final String VAULT_SECRET_ID_ENV_VAR = "VAULT_SECRET_ID"; + + /** Enable token rotation for AppRole authentication */ + public static final String VAULT_ROTATE_TOKEN_PROP = "vault.rotate-token"; + + private VaultProperties() {} +} diff --git a/hashicorp/src/test/java/org/apache/iceberg/hashicorp/TestVaultKeyManagementClient.java b/hashicorp/src/test/java/org/apache/iceberg/hashicorp/TestVaultKeyManagementClient.java new file mode 100644 index 000000000000..010ac3008b40 --- /dev/null +++ b/hashicorp/src/test/java/org/apache/iceberg/hashicorp/TestVaultKeyManagementClient.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.hashicorp; + +import static org.apache.iceberg.hashicorp.VaultProperties.VAULT_ADDRESS_PROP; +import static org.apache.iceberg.hashicorp.VaultProperties.VAULT_ROLE_ID_PROP; +import static org.apache.iceberg.hashicorp.VaultProperties.VAULT_SECRET_ID_PROP; +import static org.apache.iceberg.hashicorp.VaultProperties.VAULT_TOKEN_PROP; +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import org.apache.iceberg.TestHelpers; +import org.apache.iceberg.encryption.KeyManagementClient; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.vault.VaultContainer; + +@Testcontainers +class TestVaultKeyManagementClient { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static final String ICEBERG_TEST_KEY_NAME = "iceberg-test-key"; + private static final String VAULT_TOKEN = "root-token"; + private static final String APPROLE_ROLE_NAME = "iceberg-role"; + + @Container + @SuppressWarnings("resource") + private static final VaultContainer VAULT = + new VaultContainer<>("hashicorp/vault:1.21") + .withVaultToken(VAULT_TOKEN) + .withInitCommand( + "secrets enable transit", + "write -f transit/keys/" + ICEBERG_TEST_KEY_NAME, + "auth enable approle", + "write sys/policy/transit-policy policy='path \"transit/*\" { capabilities = [\"create\", \"read\", \"update\", \"delete\", \"list\"] }'", + "write auth/approle/role/" + APPROLE_ROLE_NAME + " token_policies=transit-policy"); + + @Test + void tokenAuthentication() { + try (KeyManagementClient client = new VaultKeyManagementClient()) { + client.initialize( + Map.of(VAULT_ADDRESS_PROP, VAULT.getHttpHostAddress(), VAULT_TOKEN_PROP, VAULT_TOKEN)); + + ByteBuffer key = ByteBuffer.wrap("table-master-key".getBytes()); + + ByteBuffer encryptedKey = client.wrapKey(key, ICEBERG_TEST_KEY_NAME); + assertThat(encryptedKey).isNotEqualTo(key); + + ByteBuffer decryptedKey = client.unwrapKey(encryptedKey, ICEBERG_TEST_KEY_NAME); + assertThat(decryptedKey).isEqualTo(key); + } + } + + @Test + void appRoleAuthentication() throws Exception { + try (KeyManagementClient client = new VaultKeyManagementClient()) { + client.initialize( + Map.of( + VAULT_ADDRESS_PROP, + VAULT.getHttpHostAddress(), + VAULT_ROLE_ID_PROP, + extractRoleId(), + VAULT_SECRET_ID_PROP, + extractSecretId())); + ByteBuffer key = ByteBuffer.wrap("appRole-authenticated-key".getBytes()); + + ByteBuffer encryptedKey = client.wrapKey(key, ICEBERG_TEST_KEY_NAME); + assertThat(encryptedKey).isNotEqualTo(key); + + ByteBuffer decryptedKey = client.unwrapKey(encryptedKey, ICEBERG_TEST_KEY_NAME); + assertThat(decryptedKey).isEqualTo(key); + } + } + + @Test + void generateKey() { + try (KeyManagementClient client = new VaultKeyManagementClient()) { + client.initialize( + Map.of(VAULT_ADDRESS_PROP, VAULT.getHttpHostAddress(), VAULT_TOKEN_PROP, VAULT_TOKEN)); + assertThat(client.supportsKeyGeneration()).isTrue(); + + KeyManagementClient.KeyGenerationResult result = client.generateKey(ICEBERG_TEST_KEY_NAME); + + assertThat(result.key()).isNotNull(); + assertThat(result.wrappedKey()).isNotNull(); + assertThat(result.key().remaining()).isGreaterThan(0); + assertThat(result.wrappedKey().remaining()).isGreaterThan(0); + } + } + + @ParameterizedTest + @MethodSource("org.apache.iceberg.TestHelpers#serializers") + public void testSerialization( + TestHelpers.RoundTripSerializer roundTripSerializer) + throws Exception { + try (VaultKeyManagementClient client = new VaultKeyManagementClient()) { + client.initialize( + Map.of(VAULT_ADDRESS_PROP, VAULT.getHttpHostAddress(), VAULT_TOKEN_PROP, VAULT_TOKEN)); + + VaultKeyManagementClient result = roundTripSerializer.apply(client); + + ByteBuffer key = ByteBuffer.wrap("table-master-key".getBytes()); + + ByteBuffer encryptedKey = result.wrapKey(key, ICEBERG_TEST_KEY_NAME); + assertThat(encryptedKey).isNotEqualTo(key); + + ByteBuffer decryptedKey = result.unwrapKey(encryptedKey, ICEBERG_TEST_KEY_NAME); + assertThat(decryptedKey).isEqualTo(key); + } + } + + private static String extractRoleId() throws IOException, InterruptedException { + String roleIdResult = + VAULT + .execInContainer( + "vault", + "read", + "-format=json", + "auth/approle/role/" + APPROLE_ROLE_NAME + "/role-id") + .getStdout(); + + JsonNode roleIdNode = MAPPER.readTree(roleIdResult); + String roleId = roleIdNode.get("data").get("role_id").asText(); + assertThat(roleId).isNotEmpty(); + return roleId; + } + + private static String extractSecretId() throws IOException, InterruptedException { + String secretIdResult = + VAULT + .execInContainer( + "vault", + "write", + "-format=json", + "-f", + "auth/approle/role/" + APPROLE_ROLE_NAME + "/secret-id") + .getStdout(); + + JsonNode secretIdNode = MAPPER.readTree(secretIdResult); + String secretId = secretIdNode.get("data").get("secret_id").asText(); + assertThat(secretId).isNotEmpty(); + return secretId; + } +} diff --git a/settings.gradle b/settings.gradle index 70f9343a252b..12cbe54c0c2b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -33,6 +33,7 @@ include 'arrow' include 'parquet' include 'bundled-guava' include 'spark' +include 'hashicorp' include 'hive-metastore' include 'nessie' include 'gcp' @@ -59,6 +60,7 @@ project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' project(':bundled-guava').name = 'iceberg-bundled-guava' project(':spark').name = 'iceberg-spark' +project(':hashicorp').name = 'iceberg-hashicorp' project(':hive-metastore').name = 'iceberg-hive-metastore' project(':nessie').name = 'iceberg-nessie' project(':gcp').name = 'iceberg-gcp'