diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 19d1986f..7c0bd58d 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -138,6 +138,7 @@ class BootStrap { else properties.setProperty(IceOptions.START_MILLIS, "" + new DateTime(DateTimeZone.UTC).withMillisOfDay(0).withDayOfMonth(1).getMillis()); properties.setProperty(IceOptions.WORK_S3_BUCKET_NAME, prop.getProperty(IceOptions.WORK_S3_BUCKET_NAME)); + properties.setProperty(IceOptions.WORK_S3_BUCKET_REGION, prop.getProperty(IceOptions.WORK_S3_BUCKET_REGION)); properties.setProperty(IceOptions.WORK_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX)); properties.setProperty(IceOptions.CUSTOM_TAGS, prop.getProperty(IceOptions.CUSTOM_TAGS, "")); diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index f179a520..6e0c6d71 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -84,7 +84,7 @@ public static Credentials getAssumedCredentials(String accountId, String assumeR * This method must be called before all methods can be used. * @param credentialsProvider */ - public static void init(AWSCredentialsProvider credentialsProvider) { + public static void init(AWSCredentialsProvider credentialsProvider, String workS3BucketRegion) { awsCredentialsProvider = credentialsProvider; clientConfig = new ClientConfiguration(); String proxyHost = System.getProperty("https.proxyHost"); @@ -95,13 +95,11 @@ public static void init(AWSCredentialsProvider credentialsProvider) { } s3Client = new AmazonS3Client(awsCredentialsProvider, clientConfig); securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig); - if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) { - if ("global".equals(System.getProperty("EC2_REGION"))) { - s3Client.setEndpoint("s3.amazonaws.com"); - } - else { - s3Client.setEndpoint("s3-" + System.getProperty("EC2_REGION") + ".amazonaws.com"); - } + if("us-east-1".equals(workS3BucketRegion)) { + s3Client.setEndpoint("s3.amazonaws.com"); + } + else { + s3Client.setEndpoint("s3-" + workS3BucketRegion + ".amazonaws.com"); } } @@ -273,7 +271,12 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke } if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) { - s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + if("us-east-1".equals(bucketFileRegion)) { + s3Client.setEndpoint("s3.amazonaws.com"); + } + else { + s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + } } ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName()); diff --git a/src/java/com/netflix/ice/common/Config.java b/src/java/com/netflix/ice/common/Config.java index 73992e03..6d7dfbfb 100644 --- a/src/java/com/netflix/ice/common/Config.java +++ b/src/java/com/netflix/ice/common/Config.java @@ -26,6 +26,7 @@ public abstract class Config { public final String workS3BucketName; + public final String workS3BucketRegion; public final String workS3BucketPrefix; public final String localDir; public final AccountService accountService; @@ -56,6 +57,17 @@ public Config( DateTime startDate = new DateTime(Long.parseLong(properties.getProperty(IceOptions.START_MILLIS)), DateTimeZone.UTC); workS3BucketName = properties.getProperty(IceOptions.WORK_S3_BUCKET_NAME); + + String defaultRegion = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION); + if (defaultRegion == null || defaultRegion.isEmpty()) throw new IllegalArgumentException("IceOptions.BILLING_S3_BUCKET_REGION must be specified"); + + String bucketRegion = properties.getProperty(IceOptions.WORK_S3_BUCKET_REGION, defaultRegion); + if (bucketRegion.isEmpty()) bucketRegion = defaultRegion; + if (bucketRegion.contains(",")) { + bucketRegion = bucketRegion.substring(0, bucketRegion.indexOf(",")); + } + workS3BucketRegion = bucketRegion; + workS3BucketPrefix = properties.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX, "ice/"); localDir = properties.getProperty(IceOptions.LOCAL_DIR, "/mnt/ice"); @@ -67,6 +79,6 @@ public Config( this.productService = productService; this.resourceService = resourceService; - AwsUtils.init(credentialsProvider); + AwsUtils.init(credentialsProvider, workS3BucketRegion); } } diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java index bfb1ee87..3282d610 100644 --- a/src/java/com/netflix/ice/common/IceOptions.java +++ b/src/java/com/netflix/ice/common/IceOptions.java @@ -95,6 +95,12 @@ public class IceOptions { */ public static final String WORK_S3_BUCKET_NAME = "ice.work_s3bucketname"; + /** + * Region for output files s3 bucket. It should be specified for buckets using v4 validation. + * It must be specified in Config. + */ + public static final String WORK_S3_BUCKET_REGION = "ice.work_s3bucketregion"; + /** * Prefix of output files in output s3 bucket. It must be specified in Config. */ diff --git a/src/java/sample.properties b/src/java/sample.properties index e5b7077a..a9e94973 100644 --- a/src/java/sample.properties +++ b/src/java/sample.properties @@ -54,6 +54,8 @@ ice.companyName=Your Company Name # s3 bucket name where Ice can store output files. Ice must have read and write access to billing s3 bucket. ice.work_s3bucketname=work_s3bucketname +# location for the output files bucket. The working bucket takes the same region than the first billing bucket if unspecified. +#ice.work_s3bucketregion=eu-east-1 # prefix of Ice output files ice.work_s3bucketprefix=ice/