diff --git a/docs/src/main/sphinx/object-storage/file-system-s3.md b/docs/src/main/sphinx/object-storage/file-system-s3.md index 12054a722e72..60bdfbaaf0a5 100644 --- a/docs/src/main/sphinx/object-storage/file-system-s3.md +++ b/docs/src/main/sphinx/object-storage/file-system-s3.md @@ -142,6 +142,11 @@ and secret keys, STS, or an IAM role: `trino-filesystem`. * - `s3.external-id` - External ID for the IAM role trust policy when connecting to S3. +* - `s3.anonymous-access` + - Use anonymous credentials for accessing public S3 buckets without + authentication. When set to `true`, no credentials are sent with S3 requests. + This takes priority over other authentication methods when enabled. + Defaults to `false`. ::: ## Security mapping diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java index 14109f59ddad..1e1f5fcae8ad 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemConfig.java @@ -159,6 +159,7 @@ public static RetryStrategy getRetryStrategy(RetryMode retryMode) private String sseKmsKeyId; private String sseCustomerKey; private boolean useWebIdentityTokenCredentialsProvider; + private boolean anonymousAccess; private SignerType signerType; private DataSize streamingPartSize = DataSize.of(32, MEGABYTE); private boolean requesterPays; @@ -397,6 +398,19 @@ public S3FileSystemConfig setUseWebIdentityTokenCredentialsProvider(boolean useW return this; } + public boolean isAnonymousAccess() + { + return anonymousAccess; + } + + @Config("s3.anonymous-access") + @ConfigDescription("Use anonymous credentials for accessing public S3 buckets") + public S3FileSystemConfig setAnonymousAccess(boolean anonymousAccess) + { + this.anonymousAccess = anonymousAccess; + return this; + } + public String getSseCustomerKey() { return sseCustomerKey; @@ -420,6 +434,21 @@ public boolean isSseWithCustomerKeyConfigValid() return true; } + @AssertTrue(message = "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)") + public boolean isAnonymousAccessConfigValid() + { + if (anonymousAccess) { + return awsAccessKey == null && + awsSecretKey == null && + iamRole == null && + externalId == null && + stsEndpoint == null && + stsRegion == null && + !useWebIdentityTokenCredentialsProvider; + } + return true; + } + public Optional getSignerType() { return Optional.ofNullable(signerType); diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java index ef6139b92427..b8de657bff26 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3FileSystemLoader.java @@ -20,6 +20,7 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.s3.S3Context.S3SseContext; import jakarta.annotation.PreDestroy; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -162,6 +163,7 @@ private static S3ClientFactory s3ClientFactory(SdkHttpClient httpClient, OpenTel Optional staticEndpoint = Optional.ofNullable(config.getEndpoint()); boolean pathStyleAccess = config.isPathStyleAccess(); boolean useWebIdentityTokenCredentialsProvider = config.isUseWebIdentityTokenCredentialsProvider(); + boolean anonymousAccess = config.isAnonymousAccess(); Optional staticIamRole = Optional.ofNullable(config.getIamRole()); String staticRoleSessionName = config.getRoleSessionName(); String externalId = config.getExternalId(); @@ -189,7 +191,10 @@ private static S3ClientFactory s3ClientFactory(SdkHttpClient httpClient, OpenTel endpoint.map(URI::create).ifPresent(s3::endpointOverride); s3.forcePathStyle(pathStyleAccess); - if (useWebIdentityTokenCredentialsProvider) { + if (anonymousAccess) { + s3.credentialsProvider(AnonymousCredentialsProvider.create()); + } + else if (useWebIdentityTokenCredentialsProvider) { s3.credentialsProvider(WebIdentityTokenFileCredentialsProvider.builder() .asyncCredentialUpdateEnabled(true) .build()); @@ -219,6 +224,7 @@ private static S3Presigner s3PreSigner(SdkHttpClient httpClient, OpenTelemetry o Optional staticEndpoint = Optional.ofNullable(config.getEndpoint()); boolean pathStyleAccess = config.isPathStyleAccess(); boolean useWebIdentityTokenCredentialsProvider = config.isUseWebIdentityTokenCredentialsProvider(); + boolean anonymousAccess = config.isAnonymousAccess(); Optional staticIamRole = Optional.ofNullable(config.getIamRole()); String staticRoleSessionName = config.getRoleSessionName(); String externalId = config.getExternalId(); @@ -233,7 +239,10 @@ private static S3Presigner s3PreSigner(SdkHttpClient httpClient, OpenTelemetry o .pathStyleAccessEnabled(pathStyleAccess) .build()); - if (useWebIdentityTokenCredentialsProvider) { + if (anonymousAccess) { + s3.credentialsProvider(AnonymousCredentialsProvider.create()); + } + else if (useWebIdentityTokenCredentialsProvider) { s3.credentialsProvider(WebIdentityTokenFileCredentialsProvider.builder() .asyncCredentialUpdateEnabled(true) .build()); diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java index 287fd703435b..6ae19aecda19 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemConfig.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.Test; import java.util.Map; +import java.util.Set; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; @@ -60,6 +61,7 @@ public void testDefaults() .setMaxErrorRetries(20) .setSseKmsKeyId(null) .setUseWebIdentityTokenCredentialsProvider(false) + .setAnonymousAccess(false) .setSseCustomerKey(null) .setStreamingPartSize(DataSize.of(32, MEGABYTE)) .setRequesterPays(false) @@ -79,20 +81,12 @@ public void testDefaults() .setApplicationId("Trino")); } - @Test - public void testExplicitPropertyMappings() + private static void addCommonProperties(ImmutableMap.Builder builder) { - Map properties = ImmutableMap.builder() - .put("s3.aws-access-key", "abc123") - .put("s3.aws-secret-key", "secret") - .put("s3.endpoint", "endpoint.example.com") + builder.put("s3.endpoint", "endpoint.example.com") .put("s3.region", "eu-central-1") .put("s3.path-style-access", "true") - .put("s3.iam-role", "myrole") .put("s3.role-session-name", "mysession") - .put("s3.external-id", "myid") - .put("s3.sts.endpoint", "sts.example.com") - .put("s3.sts.region", "us-west-2") .put("s3.storage-class", "STANDARD_IA") .put("s3.signer-type", "Aws4Signer") .put("s3.canned-acl", "BUCKET_OWNER_FULL_CONTROL") @@ -101,7 +95,6 @@ public void testExplicitPropertyMappings() .put("s3.sse.type", "KMS") .put("s3.sse.kms-key-id", "mykey") .put("s3.sse.customer-key", "customerKey") - .put("s3.use-web-identity-token-credentials-provider", "true") .put("s3.streaming.part-size", "42MB") .put("s3.requester-pays", "true") .put("s3.max-connections", "42") @@ -117,20 +110,16 @@ public void testExplicitPropertyMappings() .put("s3.http-proxy.password", "test") .put("s3.http-proxy.preemptive-basic-auth", "true") .put("s3.application-id", "application id") - .put("s3.cross-region-access", "true") - .buildOrThrow(); + .put("s3.cross-region-access", "true"); + } - S3FileSystemConfig expected = new S3FileSystemConfig() - .setAwsAccessKey("abc123") - .setAwsSecretKey("secret") + private static S3FileSystemConfig setCommonConfig(S3FileSystemConfig config) + { + return config .setEndpoint("endpoint.example.com") .setRegion("eu-central-1") .setPathStyleAccess(true) - .setIamRole("myrole") .setRoleSessionName("mysession") - .setExternalId("myid") - .setStsEndpoint("sts.example.com") - .setStsRegion("us-west-2") .setStorageClass(STANDARD_IA) .setSignerType(Aws4Signer) .setCannedAcl(ObjectCannedAcl.BUCKET_OWNER_FULL_CONTROL) @@ -139,7 +128,6 @@ public void testExplicitPropertyMappings() .setMaxErrorRetries(12) .setSseType(S3SseType.KMS) .setSseKmsKeyId("mykey") - .setUseWebIdentityTokenCredentialsProvider(true) .setSseCustomerKey("customerKey") .setRequesterPays(true) .setMaxConnections(42) @@ -156,8 +144,53 @@ public void testExplicitPropertyMappings() .setHttpProxyPreemptiveBasicProxyAuth(true) .setCrossRegionAccessEnabled(true) .setApplicationId("application id"); + } + + @Test + public void testExplicitPropertyMappings() + { + ImmutableMap.Builder builder = ImmutableMap.builder() + .put("s3.aws-access-key", "abc123") + .put("s3.aws-secret-key", "secret") + .put("s3.iam-role", "myrole") + .put("s3.external-id", "myid") + .put("s3.sts.endpoint", "sts.example.com") + .put("s3.sts.region", "us-west-2") + .put("s3.use-web-identity-token-credentials-provider", "true"); + addCommonProperties(builder); + Map properties = builder.buildOrThrow(); + + S3FileSystemConfig expected = setCommonConfig(new S3FileSystemConfig() + .setAwsAccessKey("abc123") + .setAwsSecretKey("secret") + .setIamRole("myrole") + .setExternalId("myid") + .setStsEndpoint("sts.example.com") + .setStsRegion("us-west-2") + .setUseWebIdentityTokenCredentialsProvider(true)); + + assertFullMapping(properties, expected, Set.of("s3.anonymous-access")); + } + + @Test + public void testAnonymousAccessMapping() + { + ImmutableMap.Builder builder = ImmutableMap.builder() + .put("s3.anonymous-access", "true"); + addCommonProperties(builder); + Map properties = builder.buildOrThrow(); + + S3FileSystemConfig expected = setCommonConfig(new S3FileSystemConfig() + .setAnonymousAccess(true)); - assertFullMapping(properties, expected); + assertFullMapping(properties, expected, Set.of( + "s3.aws-access-key", + "s3.aws-secret-key", + "s3.iam-role", + "s3.external-id", + "s3.sts.endpoint", + "s3.sts.region", + "s3.use-web-identity-token-credentials-provider")); } @Test @@ -169,4 +202,102 @@ public void testSSEWithCustomerKeyValidation() "s3.sse.customer-key has to be set for server-side encryption with customer-provided key", AssertTrue.class); } + + @Test + public void testAnonymousAccessWithAccessKey() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setAwsAccessKey("test-key"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithSecretKey() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setAwsSecretKey("test-secret"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithIamRole() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setIamRole("arn:aws:iam::123456789012:role/test"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithExternalId() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setExternalId("external-id"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithStsEndpoint() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setStsEndpoint("sts.example.com"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithStsRegion() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setStsRegion("us-west-2"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithWebIdentityToken() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setUseWebIdentityTokenCredentialsProvider(true), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } + + @Test + public void testAnonymousAccessWithMultipleAuthMethods() + { + assertFailsValidation( + new S3FileSystemConfig() + .setAnonymousAccess(true) + .setAwsAccessKey("test-key") + .setAwsSecretKey("test-secret") + .setIamRole("arn:aws:iam::123456789012:role/test"), + "anonymousAccessConfigValid", + "s3.anonymous-access cannot be used with other authentication methods (s3.aws-access-key, s3.aws-secret-key, s3.iam-role, s3.external-id, s3.sts.endpoint, s3.sts.region, s3.use-web-identity-token-credentials-provider)", + AssertTrue.class); + } }