Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions docs/src/main/sphinx/object-storage/file-system-gcs.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,6 @@ Cloud Storage:

* - Property
- Description
* - `gcs.use-access-token`
- Flag to set usage of a client-provided OAuth 2.0 token to access Google
Cloud Storage. Defaults to `false`, deprecated to use `gcs.auth-type` instead.
* - `gcs.auth-type`
- Authentication type to use for Google Cloud Storage access. Default to `SERVICE_ACCOUNT`.
Supported values are:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@

import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.airlift.configuration.DefunctConfig;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import jakarta.annotation.Nullable;
import jakarta.validation.constraints.AssertFalse;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotNull;
Expand All @@ -30,6 +30,7 @@

import static io.airlift.units.DataSize.Unit.MEGABYTE;

@DefunctConfig("gcs.use-access-token")
public class GcsFileSystemConfig
{
public enum AuthType
Expand All @@ -47,8 +48,7 @@ public enum AuthType
private String projectId;
private Optional<String> endpoint = Optional.empty();

private Optional<Boolean> useGcsAccessToken = Optional.empty();
private Optional<AuthType> authType = Optional.empty();
private AuthType authType = AuthType.SERVICE_ACCOUNT;
private int maxRetries = 20;
private double backoffScaleFactor = 3.0;
private Duration maxRetryTime = new Duration(25, TimeUnit.SECONDS);
Expand Down Expand Up @@ -142,30 +142,13 @@ public GcsFileSystemConfig setEndpoint(Optional<String> endpoint)
@NotNull
public AuthType getAuthType()
{
if (useGcsAccessToken.isPresent() && useGcsAccessToken.get()) {
return AuthType.ACCESS_TOKEN;
}
return authType.orElse(AuthType.SERVICE_ACCOUNT);
return authType;
}

@Config("gcs.auth-type")
public GcsFileSystemConfig setAuthType(AuthType authType)
{
this.authType = Optional.of(authType);
return this;
}

@Deprecated
public boolean isUseGcsAccessToken()
{
return useGcsAccessToken.orElse(false);
}

@Deprecated
@Config("gcs.use-access-token")
Comment thread
ebyhr marked this conversation as resolved.
public GcsFileSystemConfig setUseGcsAccessToken(boolean useGcsAccessToken)
{
this.useGcsAccessToken = Optional.of(useGcsAccessToken);
this.authType = authType;
return this;
}

Expand Down Expand Up @@ -261,10 +244,4 @@ public boolean isRetryDelayValid()
{
return minBackoffDelay.compareTo(maxBackoffDelay) <= 0;
}

@AssertFalse(message = "Cannot set both gcs.use-access-token and gcs.auth-type")
public boolean isAuthTypeAndGcsAccessTokenConfigured()
{
return authType.isPresent() && useGcsAccessToken.isPresent();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import io.trino.filesystem.gcs.GcsFileSystemConfig.AuthType;
import jakarta.validation.constraints.AssertFalse;
import jakarta.validation.constraints.AssertTrue;
import org.junit.jupiter.api.Test;

import java.util.Map;
import java.util.Optional;
import java.util.Set;

import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
Expand All @@ -43,7 +41,6 @@ void testDefaults()
.setWriteBlockSize(DataSize.of(16, MEGABYTE))
.setPageSize(100)
.setBatchSize(100)
.setUseGcsAccessToken(false)
.setProjectId(null)
.setEndpoint(Optional.empty())
.setAuthType(AuthType.SERVICE_ACCOUNT)
Expand Down Expand Up @@ -88,44 +85,7 @@ void testExplicitPropertyMappings()
.setMinBackoffDelay(new Duration(20, MILLISECONDS))
.setMaxBackoffDelay(new Duration(20, MILLISECONDS))
.setApplicationId("application id");
assertFullMapping(properties, expected, Set.of("gcs.use-access-token"));
}

// backwards compatibility test, remove if use-access-token is removed
@Test
void testExplicitPropertyMappingsWithDeprecatedUseAccessToken()
{
Map<String, String> properties = ImmutableMap.<String, String>builder()
.put("gcs.read-block-size", "51MB")
.put("gcs.write-block-size", "52MB")
.put("gcs.page-size", "10")
.put("gcs.batch-size", "11")
.put("gcs.project-id", "project")
.put("gcs.endpoint", "http://custom.dns.org:8000")
.put("gcs.use-access-token", "true")
.put("gcs.client.max-retries", "10")
.put("gcs.client.backoff-scale-factor", "4.0")
.put("gcs.client.max-retry-time", "10s")
.put("gcs.client.min-backoff-delay", "20ms")
.put("gcs.client.max-backoff-delay", "20ms")
.put("gcs.application-id", "application id")
.buildOrThrow();

GcsFileSystemConfig expected = new GcsFileSystemConfig()
.setReadBlockSize(DataSize.of(51, MEGABYTE))
.setWriteBlockSize(DataSize.of(52, MEGABYTE))
.setPageSize(10)
.setBatchSize(11)
.setProjectId("project")
.setEndpoint(Optional.of("http://custom.dns.org:8000"))
.setUseGcsAccessToken(true)
.setMaxRetries(10)
.setBackoffScaleFactor(4.0)
.setMaxRetryTime(new Duration(10, SECONDS))
.setMinBackoffDelay(new Duration(20, MILLISECONDS))
.setMaxBackoffDelay(new Duration(20, MILLISECONDS))
.setApplicationId("application id");
assertFullMapping(properties, expected, Set.of("gcs.json-key", "gcs.json-key-file-path", "gcs.auth-type"));
assertFullMapping(properties, expected);
}

@Test
Expand All @@ -138,53 +98,5 @@ public void testValidation()
"retryDelayValid",
"gcs.client.min-backoff-delay must be less than or equal to gcs.client.max-backoff-delay",
AssertTrue.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setAuthType(AuthType.ACCESS_TOKEN)
.setUseGcsAccessToken(true),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setAuthType(AuthType.ACCESS_TOKEN)
.setUseGcsAccessToken(false),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setUseGcsAccessToken(true)
.setAuthType(AuthType.SERVICE_ACCOUNT),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setUseGcsAccessToken(false)
.setAuthType(AuthType.SERVICE_ACCOUNT),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setUseGcsAccessToken(true)
.setAuthType(AuthType.APPLICATION_DEFAULT),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);

assertFailsValidation(
new GcsFileSystemConfig()
.setUseGcsAccessToken(false)
.setAuthType(AuthType.APPLICATION_DEFAULT),
"authTypeAndGcsAccessTokenConfigured",
"Cannot set both gcs.use-access-token and gcs.auth-type",
AssertFalse.class);
}
}