Skip to content

[Java] Move GCP-related secret classes and tests from sdks/java/core to google-cloud-platform-core #40019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shunping wants to merge 13 commits into
base: master
Choose a base branch
from
Open

[Java] Move GCP-related secret classes and tests from sdks/java/core to google-cloud-platform-core #40019

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
68330b6
Move gcp related secret classes to extensions/google-cloud-platform-c…
shunping Sep 4, 2026
456aaa0
Update packages, imports, and dependencies for moved GCP secret classes
shunping Sep 4, 2026
e603be6
Move GroupByKeyIT and GcpSecretRegistarTest
shunping Sep 4, 2026
6da110f
Migrate Gcp-related secret unit tests.
shunping Sep 4, 2026
149a504
Migrate GBEK tests with gcp secrets.
shunping Sep 4, 2026
c1794cd
Fix build
shunping Sep 4, 2026
233a409
Revert the visibility change of two functions after tests are moved
shunping Sep 4, 2026
949b811
Remove unused proto dep
shunping Sep 4, 2026
fa366d5
Replace GCP Secret Manager with TestSecretRegistrar in ValidateRunner…
shunping Sep 4, 2026
d1c5160
Trigger postcommit tests for XVR test change
shunping Sep 4, 2026
4dcca94
Merge branch 'master' into secret-manager-5
shunping Sep 4, 2026
1224ebf
Fix tests.
shunping Sep 5, 2026
28b727b
Merge branch 'master' into secret-manager-5
shunping Sep 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions sdks/java/build-tools/src/main/resources/beam/checkstyle/suppressions.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,9 @@
<!-- gRPC/protobuf exceptions -->
<!-- Non-vendored gRPC/protobuf imports are allowed for files that depend on libraries that expose gRPC/protobuf in its public API -->
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*extensions.*protobuf.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*core.*GcpHsmGeneratedSecret.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*core.*GroupByEncryptedKeyTest.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*core.*GroupByKeyTest.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*core.*GroupByKeyIT.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*extensions.*gcp.*GcpHsmGeneratedSecret.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*extensions.*gcp.*GcpGroupByEncryptedKeyTest.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*extensions.*gcp.*GcpGroupByKeyIT.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*core.*ValidateRunnerXlangTest.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*extensions.*ml.*" />
<suppress id="ForbidNonVendoredGrpcProtobuf" files=".*sdk.*io.*components.*ratelimiter.*" />
Expand Down
8 changes: 0 additions & 8 deletions sdks/java/core/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,7 @@ dependencies {
shadow library.java.slf4j_api
shadow library.java.snappy_java
shadow library.java.joda_time
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
implementation library.java.gax
implementation library.java.google_cloud_kms
implementation library.java.proto_google_cloud_kms_v1
implementation library.java.google_cloud_tink
implementation library.java.google_cloud_secret_manager
implementation library.java.proto_google_cloud_secret_manager_v1
implementation library.java.protobuf_java
permitUnusedDeclared enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
provided library.java.json_org
implementation library.java.everit_json_schema
implementation library.java.guava
Expand Down
153 changes: 9 additions & 144 deletions sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByEncryptedKeyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,11 @@
*/
package org.apache.beam.sdk.transforms;

import static org.junit.Assert.assertThrows;

import com.google.cloud.secretmanager.v1.ProjectName;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretName;
import com.google.cloud.secretmanager.v1.SecretPayload;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
import java.security.SecureRandom;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
Expand All @@ -39,14 +32,11 @@
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.util.GcpHsmGeneratedSecret;
import org.apache.beam.sdk.util.GcpSecret;
import org.apache.beam.sdk.util.RawSecret;
import org.apache.beam.sdk.util.Secret;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
Expand Down Expand Up @@ -100,138 +90,9 @@ public void testGroupByKeyFakeSecret() {
p.run();
}

private static final String PROJECT_ID = "apache-beam-testing";
private static final String SECRET_ID = "gbek-test";
private static Secret gcpSecret;
private static Secret gcpHsmGeneratedSecret;
private static final String KEY_RING_ID = "gbek-test-key-ring";
private static final String KEY_ID = "gbek-test-key";

@BeforeClass
public static void setup() throws IOException {
SecretManagerServiceClient client = SecretManagerServiceClient.create();
ProjectName projectName = ProjectName.of(PROJECT_ID);
SecretName secretName = SecretName.of(PROJECT_ID, SECRET_ID);

try {
client.getSecret(secretName);
} catch (Exception e) {
com.google.cloud.secretmanager.v1.Secret secret =
com.google.cloud.secretmanager.v1.Secret.newBuilder()
.setReplication(
com.google.cloud.secretmanager.v1.Replication.newBuilder()
.setAutomatic(
com.google.cloud.secretmanager.v1.Replication.Automatic.newBuilder()
.build())
.build())
.build();
client.createSecret(projectName, SECRET_ID, secret);
byte[] secretBytes = new byte[32];
new SecureRandom().nextBytes(secretBytes);
client.addSecretVersion(
secretName,
SecretPayload.newBuilder()
.setData(ByteString.copyFrom(java.util.Base64.getUrlEncoder().encode(secretBytes)))
.build());
}
gcpSecret = new GcpSecret(secretName.toString() + "/versions/latest");

try {
com.google.cloud.kms.v1.KeyManagementServiceClient kmsClient =
com.google.cloud.kms.v1.KeyManagementServiceClient.create();
String locationId = "global";
com.google.cloud.kms.v1.KeyRingName keyRingName =
com.google.cloud.kms.v1.KeyRingName.of(PROJECT_ID, locationId, KEY_RING_ID);
com.google.cloud.kms.v1.LocationName locationName =
com.google.cloud.kms.v1.LocationName.of(PROJECT_ID, locationId);
try {
kmsClient.getKeyRing(keyRingName);
} catch (Exception e) {
kmsClient.createKeyRing(
locationName, KEY_RING_ID, com.google.cloud.kms.v1.KeyRing.newBuilder().build());
}

com.google.cloud.kms.v1.CryptoKeyName keyName =
com.google.cloud.kms.v1.CryptoKeyName.of(PROJECT_ID, locationId, KEY_RING_ID, KEY_ID);
try {
kmsClient.getCryptoKey(keyName);
} catch (Exception e) {
com.google.cloud.kms.v1.CryptoKey key =
com.google.cloud.kms.v1.CryptoKey.newBuilder()
.setPurpose(com.google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT)
.build();
kmsClient.createCryptoKey(keyRingName, KEY_ID, key);
}
gcpHsmGeneratedSecret =
new GcpHsmGeneratedSecret(
PROJECT_ID,
locationId,
KEY_RING_ID,
KEY_ID,
String.format("gbek-test-job-%d", new SecureRandom().nextInt(10000)));
// Validate we have crypto permissions or skip these tests.
gcpHsmGeneratedSecret.getSecretBytes();
} catch (Exception e) {
gcpHsmGeneratedSecret = null;
}
}

@AfterClass
public static void tearDown() throws IOException {
SecretManagerServiceClient client = SecretManagerServiceClient.create();
SecretName secretName = SecretName.of(PROJECT_ID, SECRET_ID);
client.deleteSecret(secretName);
}

@Test
@Category(NeedsRunner.class)
public void testGroupByKeyGcpSecret() {
List<KV<@Nullable String, Integer>> ungroupedPairs =
Arrays.asList(
KV.of(null, 3),
KV.of("k1", 3),
KV.of("k5", Integer.MAX_VALUE),
KV.of("k5", Integer.MIN_VALUE),
KV.of("k2", 66),
KV.of("k1", 4),
KV.of(null, 5),
KV.of("k2", -33),
KV.of("k3", 0));

PCollection<KV<String, Integer>> input =
p.apply(
Create.of(ungroupedPairs)
.withCoder(KvCoder.of(NullableCoder.of(StringUtf8Coder.of()), VarIntCoder.of())));

PCollection<KV<String, Iterable<Integer>>> output =
input.apply(GroupByEncryptedKey.<String, Integer>create(gcpSecret));

PAssert.that(output.apply("Sort", MapElements.via(new SortValues())))
.containsInAnyOrder(
KV.of("k1", Arrays.asList(3, 4)),
KV.of(null, Arrays.asList(3, 5)),
KV.of("k5", Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE)),
KV.of("k2", Arrays.asList(-33, 66)),
KV.of("k3", Arrays.asList(0)));

p.run();
}

@Test
@Category(NeedsRunner.class)
public void testGroupByKeyGcpSecretThrows() {
Secret gcpSecret = new GcpSecret("bad_path/versions/latest");
p.apply(Create.of(KV.of("k1", 1)))
.apply(GroupByEncryptedKey.<String, Integer>create(gcpSecret));
assertThrows(RuntimeException.class, () -> p.run());
}

@Test
@Category(NeedsRunner.class)
public void testGroupByKeyGcpHsmGeneratedSecret() {
if (gcpHsmGeneratedSecret == null) {
return;
}
public void testGroupByKeyRawSecret() {
List<KV<@Nullable String, Integer>> ungroupedPairs =
Arrays.asList(
KV.of(null, 3),
Expand All @@ -249,8 +110,12 @@ public void testGroupByKeyGcpHsmGeneratedSecret() {
Create.of(ungroupedPairs)
.withCoder(KvCoder.of(NullableCoder.of(StringUtf8Coder.of()), VarIntCoder.of())));

// GroupByEncryptedKey expects Secret#getSecretBytes() to return Base64-URL encoded
// bytes of a valid AES key (e.g. 32 bytes for AES-256).
byte[] secretBytes = "test-encryption-key-secret-12345".getBytes(StandardCharsets.UTF_8);
Secret rawSecret = new RawSecret(Base64.getUrlEncoder().encode(secretBytes));
PCollection<KV<String, Iterable<Integer>>> output =
input.apply(GroupByEncryptedKey.<String, Integer>create(gcpHsmGeneratedSecret));
input.apply(GroupByEncryptedKey.<String, Integer>create(rawSecret));

PAssert.that(output.apply("Sort", MapElements.via(new SortValues())))
.containsInAnyOrder(
Expand Down
112 changes: 0 additions & 112 deletions sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/GroupByKeyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,12 @@
import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
import static org.junit.Assert.assertThrows;

import com.google.cloud.secretmanager.v1.ProjectName;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretName;
import com.google.cloud.secretmanager.v1.SecretPayload;
import com.google.protobuf.ByteString;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -96,9 +90,7 @@
import org.hamcrest.Matcher;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
Expand Down Expand Up @@ -624,110 +616,6 @@ public void testLargeKeys100MB() throws Exception {
}
}

/** Tests validating GroupByKey behaviors with the gbek flag set. */
@RunWith(JUnit4.class)
public static class GbekTests extends SharedTestBase {
private static final String PROJECT_ID = "apache-beam-testing";
private static final String SECRET_ID = "gbek-test";
public static String gcpSecretVersionName;
private static String secretId;

@BeforeClass
public static void setup() throws IOException {
secretId = String.format("%s-%d", SECRET_ID, new SecureRandom().nextInt(10000));
SecretManagerServiceClient client;
try {
client = SecretManagerServiceClient.create();
} catch (IOException e) {
gcpSecretVersionName = null;
return;
}
ProjectName projectName = ProjectName.of(PROJECT_ID);
SecretName secretName = SecretName.of(PROJECT_ID, secretId);

try {
client.getSecret(secretName);
} catch (Exception e) {
com.google.cloud.secretmanager.v1.Secret secret =
com.google.cloud.secretmanager.v1.Secret.newBuilder()
.setReplication(
com.google.cloud.secretmanager.v1.Replication.newBuilder()
.setAutomatic(
com.google.cloud.secretmanager.v1.Replication.Automatic.newBuilder()
.build())
.build())
.build();
client.createSecret(projectName, secretId, secret);
byte[] secretBytes = new byte[32];
new SecureRandom().nextBytes(secretBytes);
client.addSecretVersion(
secretName,
SecretPayload.newBuilder()
.setData(ByteString.copyFrom(java.util.Base64.getUrlEncoder().encode(secretBytes)))
.build());
}
gcpSecretVersionName = secretName.toString() + "/versions/latest";
}

@AfterClass
public static void tearDown() throws IOException {
if (gcpSecretVersionName != null) {
SecretManagerServiceClient client = SecretManagerServiceClient.create();
SecretName secretName = SecretName.of(PROJECT_ID, secretId);
client.deleteSecret(secretName);
}
}

@Test
@Category(NeedsRunner.class)
public void testGroupByKeyWithValidGcpSecretOption() {
if (gcpSecretVersionName == null) {
// Skip test if we couldn't set up secret manager
return;
}
List<KV<String, Integer>> ungroupedPairs =
Arrays.asList(
KV.of("k1", 3),
KV.of("k5", Integer.MAX_VALUE),
KV.of("k5", Integer.MIN_VALUE),
KV.of("k2", 66),
KV.of("k1", 4),
KV.of("k2", -33),
KV.of("k3", 0));

PCollection<KV<String, Integer>> input =
p.apply(
Create.of(ungroupedPairs)
.withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));

p.getOptions().setGbek(String.format("type:gcpsecret;version_name:%s", gcpSecretVersionName));
PCollection<KV<String, Iterable<Integer>>> output = input.apply(GroupByKey.create());

SerializableFunction<Iterable<KV<String, Iterable<Integer>>>, Void> checker =
containsKvs(
kv("k1", 3, 4),
kv("k5", Integer.MIN_VALUE, Integer.MAX_VALUE),
kv("k2", 66, -33),
kv("k3", 0));
PAssert.that(output).satisfies(checker);
PAssert.that(output).inWindow(GlobalWindow.INSTANCE).satisfies(checker);

p.run();
}

@Test
@Category(NeedsRunner.class)
public void testGroupByKeyWithInvalidGcpSecretOption() {
if (gcpSecretVersionName == null) {
// Skip test if we couldn't set up secret manager
return;
}
p.getOptions().setGbek("type:gcpsecret;version_name:bad_path/versions/latest");
p.apply(Create.of(KV.of("k1", 1))).apply(GroupByKey.create());
assertThrows(RuntimeException.class, () -> p.run());
}
}

/** Tests validating GroupByKey behaviors with windowing. */
@RunWith(JUnit4.class)
public static class WindowTests extends SharedTestBase {
Expand Down
Loading
Loading